From f4e7e50b52d3afcd98dd175188c6bb82a6663e23 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 20 Apr 2025 14:17:30 +0200 Subject: [PATCH 01/70] WIP: eager on-import profiling line_profiler/autoprofile/eager_preimport.py[i] New module for generating a dummy module where the profiling targets are pre-imported, so that they can be added to the profiler; main functionalities: - `split_dotted_path()`: split a dotted path into a module part and an attribute part - `write_eager_import_module()`: write the text of a module which does all the supplied imports and adds them to the profiler --- line_profiler/autoprofile/eager_preimports.py | 380 ++++++++++++++++++ .../autoprofile/eager_preimports.pyi | 48 +++ 2 files changed, 428 insertions(+) create mode 100644 line_profiler/autoprofile/eager_preimports.py create mode 100644 line_profiler/autoprofile/eager_preimports.pyi diff --git a/line_profiler/autoprofile/eager_preimports.py b/line_profiler/autoprofile/eager_preimports.py new file mode 100644 index 00000000..5459fbf1 --- /dev/null +++ b/line_profiler/autoprofile/eager_preimports.py @@ -0,0 +1,380 @@ +""" +Tools for eagerly pre-importing everything as specified in +`line_profiler.autoprof.run(prof_mod=...)`. +""" +import ast +import functools +import itertools +from keyword import iskeyword +from importlib.util import find_spec +from textwrap import dedent +from warnings import warn + + +def is_dotted_path(obj): + """ + Example: + >>> assert not is_dotted_path(object()) + >>> assert is_dotted_path('foo') + >>> assert is_dotted_path('foo.bar') + >>> assert not is_dotted_path('not an identifier') + >>> assert not is_dotted_path('keyword.return.not.allowed') + """ + if not (isinstance(obj, str) and obj): + return False + for chunk in obj.split('.'): + if iskeyword(chunk) or not chunk.isidentifier(): + return False + return True + + +def get_expression(obj): + """ + Example: + >>> assert not get_expression(object()) + >>> assert not get_expression('') + >>> assert not get_expression('foo; bar') + >>> assert get_expression('foo') + >>> assert get_expression('lambda x: x') + >>> assert not get_expression('def foo(x): return x') + """ + if not (isinstance(obj, str) and obj): + return None + try: + return ast.parse(obj, mode='eval') + except SyntaxError: + return None + + +def split_dotted_path(dotted_path): + """ + Arguments: + dotted_path (str): + Dotted path indicating an import target (module, package, or + a `from ... import ...`-able name under that), or an object + accessible via (chained) attribute access thereon + + Returns: + module, target (tuple[str, Union[str, None]]): + - module: dotted path indicating the module that should be + imported + - target: dotted path indicating the chained attribute access + target on the imported module corresponding to `dotted_path`; + if the import is just a module, this is set to `None` + + Raises: + - `TypeError` if `dotted_path` is not a dotted path (Python + identifiers joined by periods) + - `ModuleNotFoundError` if a matching module cannot be found + + Example: + >>> split_dotted_path('importlib.util.find_spec') + ('importlib.util', 'find_spec') + >>> split_dotted_path('importlib.util') + ('importlib.util', None) + >>> split_dotted_path('importlib.abc.Loader.exec_module') + ('importlib.abc', 'Loader.exec_module') + >>> split_dotted_path( # doctest: +NORMALIZE_WHITESPACE + ... 'not a dotted path') + Traceback (most recent call last): + ... + TypeError: dotted_path = 'not a dotted path': + expected a dotted path (string of period-joined identifiers) + >>> split_dotted_path( # doctest: +NORMALIZE_WHITESPACE + ... 'foo.bar.baz') + Traceback (most recent call last): + ... + ModuleNotFoundError: dotted_path = 'foo.bar.baz': + none of the below looks like an importable module: + ['foo.bar.baz', 'foo.bar', 'foo'] + """ + if not is_dotted_path(dotted_path): + raise TypeError(f'dotted_path = {dotted_path!r}: ' + 'expected a dotted path ' + '(string of period-joined identifiers)') + chunks = dotted_path.split('.') + checked_locs = [] + for slicing_point in range(len(chunks), 0, -1): + module = '.'.join(chunks[:slicing_point]) + target = '.'.join(chunks[slicing_point:]) or None + try: + spec = find_spec(module) + except ImportError: + spec = None + if spec is None: + checked_locs.append(module) + continue + return module, target + raise ModuleNotFoundError(f'dotted_path = {dotted_path!r}: ' + 'none of the below looks like an importable ' + f'module: {checked_locs!r}') + + +def strip(s): + return dedent(s).strip('\n') + + +class LoadedNameFinder(ast.NodeVisitor): + """ + Find the names loaded in an AST. A name is considered to be loaded + if it appears with the context `ast.Load()` and is not an argument + of any surrounding function-definition contexts + (`def func(...): ...`, `async def func(...): ...`, or + `lambda ...: ...`). + + Example: + >>> import ast + >>> + >>> + >>> module = ''' + ... def foo(x, **k): + ... def bar(y, **z): + ... pass + ... + ... return bar(x, **{**k, 'baz': foobar}) + ... + ... spam = lambda x, *y, **z: (x, y, z, a) + ... + ... str('ham') + ... ''' + >>> names = LoadedNameFinder.find(ast.parse(module)) + >>> assert names == {'bar', 'foobar', 'a', 'str'}, names + """ + def __init__(self): + self.names = set() + self.contexts = [] + + def visit_Name(self, node): + if not isinstance(node.ctx, ast.Load): + return + name = node.id + if not any(name in ctx for ctx in self.contexts): + self.names.add(node.id) + + def _visit_func_def(self, node): + args = node.args + arg_names = { + arg.arg + for arg_list in (args.posonlyargs, args.args, args.kwonlyargs) + for arg in arg_list} + if args.vararg: + arg_names.add(args.vararg.arg) + if args.kwarg: + arg_names.add(args.kwarg.arg) + self.contexts.append(arg_names) + self.generic_visit(node) + self.contexts.pop() + + visit_FunctionDef = visit_AsyncFunctionDef = visit_Lambda = _visit_func_def + + @classmethod + def find(cls, node): + finder = cls() + finder.visit(node) + return finder.names + + +def propose_names(prefixes): + """ + Generate names based on prefixes. + + Arguments: + prefixes (Collection[str]): + String identifier prefixes + + Yields: + name (str): + String identifier + + Example: + >>> import itertools + >>> + >>> + >>> list(itertools.islice(propose_names(['func', 'f', 'foo']), + ... 10)) # doctest: +NORMALIZE_WHITESPACE + ['func', 'f', 'foo', + 'func_0', 'f0', 'foo_0', + 'func_1', 'f1', 'foo_1', + 'func_2'] + """ + prefixes = list(dict.fromkeys(prefixes)) # Preserve order + if not all(is_dotted_path(p) and '.' not in p for p in prefixes): + raise TypeError(f'prefixes = {prefixes!r}: ' + 'expected string identifiers') + # Yield all the provided prefixes + yield from prefixes + # Yield the prefixes in order with numeric suffixes + prefixes_and_patterns = [ + (prefix, ('{}{}' if len(prefix) == 1 else '{}_{}').format) + for prefix in prefixes] + for i in itertools.count(): + for prefix, pattern in prefixes_and_patterns: + yield pattern(prefix, i) + + +def write_eager_import_module(dotted_paths, stream=None, *, + adder='profile.add_imported_function_or_module'): + r""" + Write a module which autoprofiles all its imports. + + Arguments: + dotted_paths (Collection[str]): + Dotted paths (strings of period-joined identifiers) + indicating what should be profiled + stream (Union[TextIO, None]): + Optional text-mode writable file object to which to write + the module + adder (str): + Single-line string `ast.parse(mode='eval')`-able to a single + expression, indicating the callable (which is assumed to + exist in the builtin namespace by the time the module is + executed) to be called to add the profiling target + + Side effects: + - `stream` (or stdout if none) written to + - Warning issued if the module can't be located for one or more + dotted paths + + Raises: + - `TypeError` if `adder` is not a string + - `ValueError` if `adder` is a non-single-line string or is not + parsable to a single expression + - `TypeError` if `dotted_paths` is not a collection of dotted + paths + + Example: + >>> import io + >>> import textwrap + >>> import warnings + >>> + >>> + >>> def strip(s): + ... return textwrap.dedent(s).strip('\n') + ... + >>> + >>> with warnings.catch_warnings(record=True) as record: + ... with io.StringIO() as sio: + ... write_eager_import_module( + ... ['importlib.util', + ... 'foo.bar', + ... 'importlib.abc.Loader.exec_module', + ... 'importlib.abc.Loader.find_module'], + ... sio) + ... written = strip(sio.getvalue()) + ... + >>> assert written == strip(''' + ... add = profile.add_imported_function_or_module + ... failures = [] + ... + ... import importlib.abc as module + ... + ... try: + ... add(module.Loader.exec_module) + ... except AttributeError: + ... failures.append('importlib.abc.Loader.exec_module') + ... try: + ... add(module.Loader.find_module) + ... except AttributeError: + ... failures.append('importlib.abc.Loader.find_module') + ... + ... import importlib.util as module + ... + ... add(module) + ... + ... if failures: + ... import warnings + ... + ... msg = '{} target{} cannot be imported: {!r}'.format( + ... len(failures), + ... '' if len(failures) == 1 else 's', + ... failures) + ... warnings.warn(msg, stacklevel=2) + ... '''), written + >>> assert len(record) == 1 + >>> assert (record[0].message.args[0] + ... == ("1 import target cannot be resolved: " + ... "['foo.bar']")) + """ + if not isinstance(adder, str): + AdderError = TypeError + elif len(adder.splitlines()) != 1: + AdderError = ValueError + else: + expr = get_expression(adder) + if expr: + AdderError = None + else: + AdderError = ValueError + if AdderError: + raise AdderError(f'adder = {adder!r}: ' + 'expected a single-line string parsable to a single ' + 'expression') + + # Get the names loaded by `adder`; + # these names are not allowed in the namespace + forbidden_names = LoadedNameFinder.find(expr) + # We need three free names: + # - One for `adder` + # - One for a list of failed targets + # - One for the imported module + adder_name = next( + name for name in propose_names(['add', 'add_func', 'a', 'f']) + if name not in forbidden_names) + forbidden_names.add(adder_name) + failures_name = next( + name + for name in propose_names(['failures', 'failed_targets', 'f', '_']) + if name not in forbidden_names) + forbidden_names.add(failures_name) + module_name = next( + name for name in propose_names(['module', 'mod', 'imported', 'm', '_']) + if name not in forbidden_names) + + # Figure out the import targets to profile + imports = {} + unknown_locs = [] + for path in sorted(set(dotted_paths)): + try: + module, target = split_dotted_path(path) + except ModuleNotFoundError: + unknown_locs.append(path) + continue + imports.setdefault(module, []).append(target) + + # Warn against failed imports + if unknown_locs: + msg = '{} import target{} cannot be resolved: {!r}'.format( + len(unknown_locs), + '' if len(unknown_locs) == 1 else 's', + unknown_locs) + warn(msg, stacklevel=2) + + # Do the imports and add them with `adder` + write = functools.partial(print, file=stream) + write(f'{adder_name} = {adder}\n{failures_name} = []') + for module, targets in imports.items(): + write(f'\nimport {module} as {module_name}\n') + for target in targets: + if target is None: + write(f'{adder_name}({module_name})') + continue + path = f'{module}.{target}' + write(strip(f""" + try: + {adder_name}({module_name}.{target}) + except AttributeError: + {failures_name}.append({path!r}) + """)) + # Issue a warning if any of the targets doesn't exist + if imports: + write('') + write(strip(f""" + if {failures_name}: + import warnings + + msg = '{{}} target{{}} cannot be imported: {{!r}}'.format( + len({failures_name}), + '' if len({failures_name}) == 1 else 's', + {failures_name}) + warnings.warn(msg, stacklevel=2) + """)) diff --git a/line_profiler/autoprofile/eager_preimports.pyi b/line_profiler/autoprofile/eager_preimports.pyi new file mode 100644 index 00000000..5f312f18 --- /dev/null +++ b/line_profiler/autoprofile/eager_preimports.pyi @@ -0,0 +1,48 @@ +import ast +from typing import ( + Any, Collection, Generator, List, Set, TextIO, Tuple, Union) + + +def is_dotted_path(obj: Any) -> bool: + ... + + +def get_expression(obj: Any) -> Union[ast.Expression, None]: + ... + + +def split_dotted_path(dotted_path: str) -> Tuple[str, Union[str, None]]: + ... + + +def strip(s: str) -> str: + ... + + +class LoadedNameFinder(ast.NodeVisitor): + names: Set[str] + contexts: List[Set[str]] + + def visit_Name(self, node: ast.Name) -> None: + ... + + def visit_FunctionDef(self, + node: Union[ast.FunctionDef, ast.AsyncFunctionDef, + ast.Lambda]) -> None: + ... + + visit_AsyncFunctionDef = visit_Lambda = visit_FunctionDef + + @classmethod + def find(cls, node: ast.AST) -> Set[str]: + ... + + +def propose_names(prefixes: Collection[str]) -> Generator[str, None, None]: + ... + + +def write_eager_import_module( + dotted_paths: Collection[str], stream: Union[TextIO, None] = None, *, + adder: str = 'profile.add_imported_function_or_module') -> None: + ... From 27836400f3141109f89aa2995d52b141e8386037 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 20 Apr 2025 17:08:16 +0200 Subject: [PATCH 02/70] Tests: `~.autoprofile.eager_preimports` tests/test_eager_preimports.py create_doctest_wrapper(), regularize_doctests() New functions to create hooks running doctests, even when `--doctest-modules` or `--xdoctest` is not passed test_doctest_*() Hook tests for the `line_profiler.autoprofile.eager_preimports` doctest test_write_eager_import_module_wrong_adder() Test for passing bad `adder` values to `write_eager_import_module()` --- tests/test_eager_preimports.py | 171 +++++++++++++++++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 tests/test_eager_preimports.py diff --git a/tests/test_eager_preimports.py b/tests/test_eager_preimports.py new file mode 100644 index 00000000..b070c63f --- /dev/null +++ b/tests/test_eager_preimports.py @@ -0,0 +1,171 @@ +""" +Tests for `line_profiler.autoprofile.eager_preimports`. + +Notes +----- +Most of the features are already covered by the doctests, but this +project doesn't generally use the `--doctest-modules` option. So this is +mostly a hook to run the doctests. +""" +import doctest +import functools +import importlib +import pathlib +from types import ModuleType +from typing import Any, Callable, Dict, Type, Union + +import pytest +from _pytest import doctest as pytest_doctest + +from line_profiler.autoprofile import eager_preimports, util_static + + +CAN_USE_PYTEST_DOCTEST = True + +try: + class PytestDoctestRunner(pytest_doctest._init_runner_class()): + # Neuter these methods because they expect `out` to be a + # callable while `pytest` passes a list + + def report_start(self, out, *args, **kwargs): + pass + + def report_success(self, out, *args, **kwargs): + pass +except Exception: + CAN_USE_PYTEST_DOCTEST = False + + +def create_doctest_wrapper( + test: doctest.DocTest, *, + fname: Union[str, pathlib.PurePath, None] = None, + globs: Union[Dict[str, Any], None] = None, + name: Union[str, None] = None, + strip_prefix: Union[str, None] = None, + test_name_prefix: str = 'test_doctest_', + use_pytest_doctest: bool = CAN_USE_PYTEST_DOCTEST) -> Callable: + """ + Create a hook to run a doctest as if it was a regular test. + + Returns + wrapper (Callable): + Test function + """ + if strip_prefix is not None: + assert test.name.startswith(strip_prefix) + bare_name = test.name[len(strip_prefix):].lstrip('.').replace('.', '_') + if not bare_name: + bare_name = strip_prefix + else: + bare_name = test.name.replace('.', '_') + if name is None: + name = test_name_prefix + bare_name + assert name.isidentifier() + if fname is not None: + fname = pathlib.Path(fname) + + use_pytest_doctest = bool(use_pytest_doctest) & CAN_USE_PYTEST_DOCTEST + try: + item_from_parent = pytest_doctest.DoctestItem.from_parent + module_from_parent = pytest_doctest.DoctestModule.from_parent + get_doctest_option_flags = pytest_doctest.get_optionflags + xc_to_info = pytest.ExceptionInfo.from_exception + checker = pytest_doctest._get_checker() + except Exception: + use_pytest_doctest = False + + def wrapper_pytest(request: pytest.FixtureRequest) -> None: + if globs is not None: + test.globs = globs.copy() + module = module_from_parent(parent=request.session, path=fname) + runner = PytestDoctestRunner( + checker=checker, + optionflags=get_doctest_option_flags(request.config), + continue_on_failure=False) + item = item_from_parent(module, name=name, runner=runner, dtest=test) + item.setup() + try: + item.runtest() + except doctest.UnexpectedException as e: + msg = '{}\n{}'.format(e.exc_info[0].__name__, + item.repr_failure(xc_to_info(e))) + raise pytest.fail(msg) from None + + def wrapper_vanilla() -> None: + if globs is not None: + test.globs = globs.copy() + runner = doctest.DebugRunner() + runner.run(test) + + if use_pytest_doctest: + wrapper = wrapper_pytest + else: + wrapper = wrapper_vanilla + + doctest_backend = '_pytest.doctest' if use_pytest_doctest else 'doctest' + wrapper.__name__ = name + wrapper.__doc__ = ('Run the doctest for `{}` with the facilities of `{}`' + .format(bare_name, doctest_backend)) + return wrapper + + +def regularize_doctests( + obj: Any, *, + namespace: Union[Dict[str, Any], None] = None, + finder: Union[doctest.DocTestFinder, None] = None, + strip_common_prefix: bool = True, + use_pytest_doctest: bool = CAN_USE_PYTEST_DOCTEST) -> Dict[str, + Callable]: + """ + Gather doctests from `obj` and make them regular test functions. + + Returns: + wrappers (dict[str, Callable]): + Dictionary from test names to Test functions + """ + if isinstance(obj, ModuleType): + prefix = module = obj.__name__ + fname = obj.__file__ + globs = vars(obj) + else: + module = obj.__module__ + prefix = f'{module}.{obj.__qualname__}' + fname = util_static.modname_to_modpath(module) + globs = vars(importlib.import_module(module)) + + if finder is None: + finder = doctest.DocTestFinder() + + make_wrapper = functools.partial( + create_doctest_wrapper, + fname=fname, globs=globs, strip_prefix=prefix, + use_pytest_doctest=use_pytest_doctest) + + tests = [make_wrapper(test) for test in finder.find(obj) if test.examples] + result = {test.__name__: test for test in tests} + if namespace is None: + return result + for name, test in result.items(): + if name in namespace: + test_module = getattr(namespace.get('__spec__'), 'name', '???') + raise AttributeError(f'module `{test_module}` already has a test ' + f'(or other entity) named `{name}()`') + namespace[name] = test + return result + + +@pytest.mark.parametrize( + 'adder, xc, msg', + [('foo; bar', ValueError, None), + (1, TypeError, None), + ('(foo\n .bar)', ValueError, None)]) +def test_write_eager_import_module_wrong_adder( + adder: Any, xc: Type[Exception], msg: Union[str, None]) -> None: + """ + Test passing an erroneous `adder` to `write_eager_import_module()`. + """ + with pytest.raises(xc, match=msg): + eager_preimports.write_eager_import_module(['foo'], adder=adder) + + +regularize_doctests(eager_preimports, namespace=globals()) From 0a9fc7268eb7555d682ae8603bc34f9df89855d8 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 20 Apr 2025 20:36:36 +0200 Subject: [PATCH 03/70] Eager pre-importing in `kernprof` kernprof.py __doc__ Updated with the new option main() Added new option `-e`/`--eager-preimports` for eagerly importing the `--prof-mod` targets, so that they are all unconditionally profiled (where possible) regardless of whether they are imported in the test script/module --- kernprof.py | 104 +++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 94 insertions(+), 10 deletions(-) diff --git a/kernprof.py b/kernprof.py index 39bbfde5..038ebb08 100755 --- a/kernprof.py +++ b/kernprof.py @@ -91,6 +91,9 @@ def main(): -p, --prof-mod PROF_MOD List of modules, functions and/or classes to profile specified by their name or path. List is comma separated, adding the current script path profiles the full script. Multiple copies of this flag can be supplied and the.list is extended. Only works with line_profiler -l, --line-by-line + -e, --eager-preimports + If specified, all modules, classes, etc. specified in `--prof-mod` will be imported and marked for profiling before running the script/module, + regardless of whether they are directly imported in the script/module. Only works with line_profiler -l, --line-by-line --prof-imports If specified, modules specified to `--prof-mod` will also autoprofile modules that they import. Only works with line_profiler -l, --line-by-line """ import builtins @@ -102,6 +105,8 @@ def main(): import concurrent.futures # NOQA import tempfile import time +import traceback +import warnings from argparse import ArgumentError, ArgumentParser from runpy import run_module @@ -416,6 +421,13 @@ def positive_float(value): "List is comma separated, adding the current script path profiles the full script. " "Multiple copies of this flag can be supplied and the.list is extended. " "Only works with line_profiler -l, --line-by-line") + parser.add_argument('-e', '--eager-preimports', action='store_true', + help="If specified, all modules, classes, etc. " + "specified in `--prof-mod` will be imported and " + "marked for profiling before running the script/" + "module, regardless of whether they are directly " + "imported in the script/module. " + "Only works with line_profiler -l, --line-by-line") parser.add_argument('--prof-imports', action='store_true', help="If specified, modules specified to `--prof-mod` will also autoprofile modules that they import. " "Only works with line_profiler -l, --line-by-line") @@ -550,6 +562,86 @@ def _main(options, module=False): __file__ = script_file __name__ = '__main__' + # If using eager pre-imports, write a dummy module which contains + # all those imports and marks them for profiling, then run it + if options.prof_mod: + # Note: `prof_mod` entries can be filenames (which can contain + # commas), so check against existing filenames before splitting + # them + options.prof_mod = sum( + ([spec] if os.path.exists(spec) else spec.split(',') + for spec in options.prof_mod), + []) + if options.line_by_line and options.prof_mod and options.eager_preimports: + # We assume most items in `.prof_mod` to be import-able without + # significant side effects, but the same cannot be said if it + # contains the script file to be run -- so don't eager-import + # it. + from line_profiler.autoprofile.eager_preimports import ( + is_dotted_path, propose_names, write_eager_import_module) + from line_profiler.autoprofile.util_static import modpath_to_modname + from line_profiler.autoprofile.autoprofile import ( + _extend_line_profiler_for_profiling_imports as upgrade_profiler) + + filtered_targets = [] + invalid_targets = [] + for target in options.prof_mod: + if is_dotted_path(target): + filtered_targets.append(target) + continue + try: + with open(os.devnull, mode='w') as fobj: + with contextlib.redirect_stderr(fobj): + filename = find_script(target) + except SystemExit: # No such file + invalid_targets.append(target) + continue + if not module and os.path.samefile(filename, script_file): + # Ignore the script to be run in eager importing + continue + modname = modpath_to_modname(filename) + if modname is None: # Not import-able + invalid_targets.append(target) + continue + filtered_targets.append(modname) + if invalid_targets: + invalid_targets = sorted(set(invalid_targets)) + msg = ('{} profile-on-import target{} cannot be converted to ' + 'dotted-path form: {!r}' + .format(len(invalid_targets), + '' if len(invalid_targets) == 1 else 's', + invalid_targets)) + warnings.warn(msg) + if filtered_targets: + # - We could've done everything in-memory with `io.StringIO` + # and `exec()`, but that results in indecipherable + # tracebacks should anything goes wrong; + # so we write to a tempfile and `execfile()` it + # - While this works theoretically for preserving traceback, + # the catch is that the tempfile will already have been + # deleted by the time the traceback is formatted; + # so we have to format the traceback and manually print + # the context before re-raising the error + upgrade_profiler(prof) + temp_mod_name = next( + name for name in propose_names(['_kernprof_eager_preimports']) + if name not in sys.modules) + with tempfile.TemporaryDirectory() as tmpdir: + temp_mod_path = os.path.join(tmpdir, temp_mod_name + '.py') + with open(temp_mod_path, mode='w') as fobj: + write_eager_import_module(filtered_targets, stream=fobj) + ns = {} # Use a fresh namespace + try: + execfile(temp_mod_path, ns, ns) + except Exception as e: + tb_lines = traceback.format_tb(e.__traceback__) + i_last_temp_frame = max( + i for i, line in enumerate(tb_lines) + if temp_mod_path in line) + print('\nContext:', ''.join(tb_lines[i_last_temp_frame:]), + end='', sep='\n', file=sys.stderr) + raise + if options.output_interval: rt = RepeatedTimer(max(options.output_interval, 1), prof.dump_stats, options.outfile) original_stdout = sys.stdout @@ -563,16 +655,8 @@ def _main(options, module=False): ns = locals() if options.prof_mod and options.line_by_line: from line_profiler.autoprofile import autoprofile - # Note: `prof_mod` entries can be filenames (which can - # contain commas), so check against existing filenames - # before splitting them - prof_mod = sum( - ([spec] if os.path.exists(spec) else spec.split(',') - for spec in options.prof_mod), - []) - autoprofile.run(script_file, - ns, - prof_mod=prof_mod, + autoprofile.run(script_file, ns, + prof_mod=options.prof_mod, profile_imports=options.prof_imports, as_module=module is not None) elif module and options.builtin: From a5017f3ab18df9d3deecd9e3da39b156cf78c2ea Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 20 Apr 2025 22:35:46 +0200 Subject: [PATCH 04/70] Extended `--eager-preimports` kernprof.py __doc__ Updated find_script() Added argument `exit_on_error` for when it should raise an error instead of quitting when a file is not found _normalize_profiling_targets() - Moved code for normalizing `--prof-mod` from `main()` to here - Added path resolution with `find_script()` and `os.path.abspath()` main() - Updated help texts for `-p`/`--prof-mod` and `-e`/`--eager-preimports` - Now permitting using `-e`/`--eager-preimports` with arguments, as with `-p`/`--prof-mod`; the no-arg form corrsponds to just taking the `--prof-mod`, while the with-arg form permits specifying different profiling targets between the two options --- kernprof.py | 130 +++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 94 insertions(+), 36 deletions(-) diff --git a/kernprof.py b/kernprof.py index 038ebb08..8518d197 100755 --- a/kernprof.py +++ b/kernprof.py @@ -89,11 +89,13 @@ def main(): Enables outputting of cumulative profiling results to file every n seconds. Uses the threading module. Minimum value is 1 (second). Defaults to disabled. -p, --prof-mod PROF_MOD - List of modules, functions and/or classes to profile specified by their name or path. List is comma separated, adding the current script path profiles - the full script. Multiple copies of this flag can be supplied and the.list is extended. Only works with line_profiler -l, --line-by-line + List of modules, functions and/or classes to profile specified by their name or path, if they are imported in the profiled script/module. These + profiling targets can be supplied both as comma-separated items, or separately with multiple copies of this flag. Adding the current script/module + profiles the entirety of it. Only works with line_profiler -l, --line-by-line. -e, --eager-preimports - If specified, all modules, classes, etc. specified in `--prof-mod` will be imported and marked for profiling before running the script/module, - regardless of whether they are directly imported in the script/module. Only works with line_profiler -l, --line-by-line + List of modules, functions, and/or classes, to be imported and marked for profiling before running the script/module, regardless of whether they are + directly imported in the script/module. Follows the same semantics as `--prof-mod`. If supplied without an argument, indicates that all `--prof-mod` + targets are to be so profiled. Only works with line_profiler -l, --line-by-line. --prof-imports If specified, modules specified to `--prof-mod` will also autoprofile modules that they import. Only works with line_profiler -l, --line-by-line """ import builtins @@ -211,7 +213,7 @@ def find_module_script(module_name): raise SystemExit(1) -def find_script(script_name): +def find_script(script_name, exit_on_error=True): """ Find the script. If the input is not a file, then $PATH will be searched. @@ -226,8 +228,12 @@ def find_script(script_name): if os.path.isfile(fn): return fn - sys.stderr.write('Could not find script %s\n' % script_name) - raise SystemExit(1) + msg = f'Could not find script {script_name!r}' + if exit_on_error: + print(msg, file=sys.stderr) + raise SystemExit(1) + else: + raise FileNotFoundError(msg) def _python_command(): @@ -241,6 +247,34 @@ def _python_command(): return sys.executable +def _normalize_profiling_targets(targets): + """ + Normalize the parsed `--prof-mod` and `--eager-preimports` by: + - Normalizing file paths with `find_script()`, and subsequently + to absolute paths. + - Splitting non-file paths at commas into (presumably) file paths + and/or dotted paths. + - Removing duplicates. + """ + def find(path): + try: + path = find_script(path, exit_on_error=False) + except FileNotFoundError: + return None + return os.path.abspath(path) + + results = {} + for chunk in targets: + filename = find(chunk) + if filename is not None: + results.setdefault(filename) + continue + for subchunk in chunk.split(','): + filename = find(subchunk) + results.setdefault(subchunk if filename is None else filename) + return list(results) + + class _restore_list: """ Restore a list like `sys.path` after running code which potentially @@ -416,18 +450,34 @@ def positive_float(value): parser.add_argument('-i', '--output-interval', type=int, default=0, const=0, nargs='?', help="Enables outputting of cumulative profiling results to file every n seconds. Uses the threading module. " "Minimum value is 1 (second). Defaults to disabled.") - parser.add_argument('-p', '--prof-mod', action='append', type=str, - help="List of modules, functions and/or classes to profile specified by their name or path. " - "List is comma separated, adding the current script path profiles the full script. " - "Multiple copies of this flag can be supplied and the.list is extended. " - "Only works with line_profiler -l, --line-by-line") - parser.add_argument('-e', '--eager-preimports', action='store_true', - help="If specified, all modules, classes, etc. " - "specified in `--prof-mod` will be imported and " - "marked for profiling before running the script/" - "module, regardless of whether they are directly " - "imported in the script/module. " - "Only works with line_profiler -l, --line-by-line") + parser.add_argument('-p', '--prof-mod', + action='append', + metavar=("{path/to/script | object.dotted.path}" + "[,...]"), + help="List of modules, functions and/or classes " + "to profile specified by their name or path, " + "if they are imported in the profiled " + "script/module. " + "These profiling targets can be supplied both as " + "comma-separated items, or separately with " + "multiple copies of this flag. " + "Adding the current script/module profiles the " + "entirety of it. " + "Only works with line_profiler -l, --line-by-line.") + parser.add_argument('-e', '--eager-preimports', + action='append', + const=True, + metavar=("{path/to/script | object.dotted.path}" + "[,...]"), + nargs='?', + help="List of modules, functions, and/or classes, " + "to be imported and marked for profiling before " + "running the script/module, regardless of whether " + "they are directly imported in the script/module. " + "Follows the same semantics as `--prof-mod`. " + "If supplied without an argument, indicates that " + "all `--prof-mod` targets are to be so profiled. " + "Only works with line_profiler -l, --line-by-line.") parser.add_argument('--prof-imports', action='store_true', help="If specified, modules specified to `--prof-mod` will also autoprofile modules that they import. " "Only works with line_profiler -l, --line-by-line") @@ -568,15 +618,21 @@ def _main(options, module=False): # Note: `prof_mod` entries can be filenames (which can contain # commas), so check against existing filenames before splitting # them - options.prof_mod = sum( - ([spec] if os.path.exists(spec) else spec.split(',') - for spec in options.prof_mod), - []) - if options.line_by_line and options.prof_mod and options.eager_preimports: - # We assume most items in `.prof_mod` to be import-able without - # significant side effects, but the same cannot be said if it - # contains the script file to be run -- so don't eager-import - # it. + options.prof_mod = _normalize_profiling_targets(options.prof_mod) + if options.eager_preimports: + if options.eager_preimports == [True]: + # Eager-import all of `--prof-mod` + options.eager_preimports = list(options.prof_mod or []) + else: # Only eager-import the specified targets + options.eager_preimports = _normalize_profiling_targets([ + target for target in options.eager_preimports + if target not in (True,)]) + if options.line_by_line and options.eager_preimports: + # We assume most items in `.eager_preimports` to be import-able + # without significant side effects, but the same cannot be said + # if it contains the script file to be run. E.g. the script may + # not even have a `if __name__ == '__main__': ...` guard. So + # don't eager-import it. from line_profiler.autoprofile.eager_preimports import ( is_dotted_path, propose_names, write_eager_import_module) from line_profiler.autoprofile.util_static import modpath_to_modname @@ -585,21 +641,23 @@ def _main(options, module=False): filtered_targets = [] invalid_targets = [] - for target in options.prof_mod: + for target in options.eager_preimports: if is_dotted_path(target): filtered_targets.append(target) continue - try: - with open(os.devnull, mode='w') as fobj: - with contextlib.redirect_stderr(fobj): - filename = find_script(target) - except SystemExit: # No such file + # Filenames are already normalized in + # `_normalize_profiling_targets()` + if not os.path.exists(target): invalid_targets.append(target) continue - if not module and os.path.samefile(filename, script_file): + if not module and os.path.samefile(target, script_file): # Ignore the script to be run in eager importing + # (but make sure that it is handled by `--prof-mod`) + if options.prof_mod is None: + options.prof_mod = [] + options.prof_mod.append(script_file) continue - modname = modpath_to_modname(filename) + modname = modpath_to_modname(target) if modname is None: # Not import-able invalid_targets.append(target) continue From 51484cb1383921dd6eb0f81011d1a3019b72154f Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 21 Apr 2025 01:29:34 +0200 Subject: [PATCH 05/70] More aggressive on-import profiling line_profiler/autoprofile/line_profiler_utils.py add_imported_function_or_module() - Added new argument `wrap` for controlling whether to replace class and module members with wrappers - Refactored object adding to be more aggressive, ditching the explicit `inspect.isfunction()` check (since we expanded the catalog of addable objects in #332) - Now returning whether any function has been added to the profiler line_profiler/line_profiler.py add_module() - Now shares an implementation with `.add_class()` - Added new argument `wrap` for controlling whether to replace members with wrappers - Refactored object adding to be more aggressive, ditching the explicit `is_function()` check (since we expanded the catalog of addable objects in #332) add_class() New method (alias to `.add_module()`) --- .../autoprofile/line_profiler_utils.py | 46 ++++++++++------ line_profiler/line_profiler.py | 53 +++++++++++++------ 2 files changed, 66 insertions(+), 33 deletions(-) diff --git a/line_profiler/autoprofile/line_profiler_utils.py b/line_profiler/autoprofile/line_profiler_utils.py index d7f63bc3..cbd6e6c2 100644 --- a/line_profiler/autoprofile/line_profiler_utils.py +++ b/line_profiler/autoprofile/line_profiler_utils.py @@ -1,25 +1,39 @@ import inspect -def add_imported_function_or_module(self, item): - """Method to add an object to LineProfiler to be profiled. +def add_imported_function_or_module(self, item, *, wrap=False): + """ + Method to add an object to `LineProfiler` to be profiled. - This method is used to extend an instance of LineProfiler so it can identify - whether an object is function/method, class or module and handle it's - profiling accordingly. + This method is used to extend an instance of `LineProfiler` so it + can identify whether an object is a callable (wrapper), a class, or + a module, and handle its profiling accordingly. Args: - item (Callable | Type | ModuleType): - object to be profiled. + item (Union[Callable, Type, ModuleType]): + Object to be profiled. + wrap (bool): + Whether to replace the wrapped members with wrappers which + automatically enable/disable the profiler when called. + + Returns: + 1 if any function is added to the profiler, 0 otherwise. + + See also: + `LineProfiler.add_callable()`, `.add_module()`, `.add_class()` """ - if inspect.isfunction(item): - self.add_function(item) - elif inspect.isclass(item): - for k, v in item.__dict__.items(): - if inspect.isfunction(v): - self.add_function(v) + if inspect.isclass(item): + count = self.add_class(item, wrap=wrap) elif inspect.ismodule(item): - self.add_module(item) + count = self.add_module(item, wrap=wrap) else: - return - self.enable_by_count() + try: + count = self.add_callable(item) + except TypeError: + count = 0 + if count: + # Session-wide enabling means that we no longer have to wrap + # individual callables to enable/disable the profiler when + # they're called + self.enable_by_count() + return 1 if count else 0 diff --git a/line_profiler/line_profiler.py b/line_profiler/line_profiler.py index 0e0fd74c..fd1cb8e7 100755 --- a/line_profiler/line_profiler.py +++ b/line_profiler/line_profiler.py @@ -116,7 +116,7 @@ def add_callable(self, func): the underlying Cython profiler. Returns: - 1 if any function is added to the profiler, 0 otherwise + 1 if any function is added to the profiler, 0 otherwise. """ nadded = 0 for impl in _get_underlying_functions(func): @@ -149,23 +149,42 @@ def print_stats(self, stream=None, output_unit=None, stripzeros=False, stream=stream, stripzeros=stripzeros, details=details, summarize=summarize, sort=sort, rich=rich) - def add_module(self, mod): - """ Add all the functions in a module and its classes. + def _add_namespace(self, namespace, *, wrap=False): """ - from inspect import isclass - - nfuncsadded = 0 - for item in mod.__dict__.values(): - if isclass(item): - for k, v in item.__dict__.items(): - if is_function(v): - self.add_function(v) - nfuncsadded += 1 - elif is_function(item): - self.add_function(item) - nfuncsadded += 1 - - return nfuncsadded + Add the members (callables (wrappers), methods, classes, ...) in + a namespace and profile them. + + Args: + namespace (Union[ModuleType, type]): + Module or class to be profiled. + wrap (bool): + Whether to replace the wrapped members with wrappers + which automatically enable/disable the profiler when + called. + + Returns: + n (int): + Number of members added to the profiler. + """ + count = 0 + add_cls = self.add_class + add_func = self.add_callable + for attr, value in vars(namespace).items(): + if isinstance(value, type): + count += 1 if add_cls(value, wrap=wrap) else 0 + continue + try: + func_needs_adding = add_func(value) + except TypeError: # Not a callable (wrapper) + continue + if not func_needs_adding: + continue + if wrap: + setattr(namespace, attr, self.wrap_callable(value)) + count += 1 + return count + + add_class = add_module = _add_namespace def _get_wrapper_info(self, func): info = getattr(func, self._profiler_wrapped_marker, None) From 9cf3f952eacfbc2a7756a4c3af20fc3627eca83c Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 21 Apr 2025 02:01:26 +0200 Subject: [PATCH 06/70] Stub files --- line_profiler/autoprofile/line_profiler_utils.pyi | 6 +++--- line_profiler/line_profiler.pyi | 6 +++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/line_profiler/autoprofile/line_profiler_utils.pyi b/line_profiler/autoprofile/line_profiler_utils.pyi index 467c10e7..11d380bc 100644 --- a/line_profiler/autoprofile/line_profiler_utils.pyi +++ b/line_profiler/autoprofile/line_profiler_utils.pyi @@ -1,8 +1,8 @@ -from typing import Callable -from typing import Type +from typing import Callable, Literal from types import ModuleType def add_imported_function_or_module( - self, item: Callable | Type | ModuleType) -> None: + self, item: Callable | type | ModuleType, *, + wrap: bool = False) -> Literal[0, 1]: ... diff --git a/line_profiler/line_profiler.pyi b/line_profiler/line_profiler.pyi index edbf1bde..1607e5d5 100644 --- a/line_profiler/line_profiler.pyi +++ b/line_profiler/line_profiler.pyi @@ -1,3 +1,4 @@ +from types import ModuleType from typing import Literal, List, Tuple import io from ._line_profiler import LineProfiler as CLineProfiler @@ -26,7 +27,10 @@ class LineProfiler(CLineProfiler, ByCountProfilerMixin): rich: bool = ...) -> None: ... - def add_module(self, mod) -> int: + def add_module(self, mod: ModuleType, *, wrap: bool = False) -> int: + ... + + def add_class(self, cls: type, *, wrap: bool = False) -> int: ... From 4b16b8655a919ecea4baa419b257b38a7c6a7732 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 21 Apr 2025 08:51:17 +0200 Subject: [PATCH 07/70] Test for `--eager-preimports` tests/test_autoprofile.py test_autoprofile_eager_preimports() New test for the behaviors of the `-e` and `-p` options test_autoprofile_callable_wrapper_objects() Test that on-import autoprofiling catches callable wrapper types like classmethods --- tests/test_autoprofile.py | 148 +++++++++++++++++++++++++++++++++++++- 1 file changed, 147 insertions(+), 1 deletion(-) diff --git a/tests/test_autoprofile.py b/tests/test_autoprofile.py index 839038a8..94ed7b14 100644 --- a/tests/test_autoprofile.py +++ b/tests/test_autoprofile.py @@ -612,7 +612,6 @@ def test_autoprofile_from_inlined_script(outfile, expected_outfile) -> None: print(proc.stdout) print(proc.stderr) proc.check_returncode() - outfile, = temp_dpath.glob(expected_outfile) lp_cmd = [sys.executable, '-m', 'line_profiler', str(outfile)] proc = ub.cmd(lp_cmd) @@ -623,3 +622,150 @@ def test_autoprofile_from_inlined_script(outfile, expected_outfile) -> None: assert 'Function: add_one' in raw_output assert 'Function: add_two' not in raw_output assert 'Function: add_three' in raw_output + + +@pytest.mark.parametrize( + ['prof_mod', 'eager_preimports', + 'add_one', 'add_two', 'add_three', 'add_four', 'add_operator', 'main'], + # Test that `--eager-preimports` know to exclude the script run + # (so as not to inadvertantly run it twice) + [('script.py', None, False, False, False, False, False, True), + (None, 'script.py', False, False, False, False, False, True), + # Test explicitly passing targets to `--eager-preimports` + (['test_mod.submod1,test_mod.submod2', 'test_mod.subpkg.submod4'], None, + True, True, False, False, True, False), + (['test_mod.submod1,test_mod.submod2'], ['test_mod.subpkg.submod4'], + True, True, False, True, True, False), + (None, ['test_mod.submod1,test_mod.submod2', 'test_mod.subpkg.submod4'], + True, True, False, True, True, False), + # Test implicitly passing targets to `--eager-preimports` + (['test_mod.submod1,test_mod.submod2', 'test_mod.subpkg.submod4'], True, + True, True, False, True, True, False)]) +def test_autoprofile_eager_preimports( + prof_mod, eager_preimports, + add_one, add_two, add_three, add_four, add_operator, main): + """ + Test eager imports with the `-e`/`--eager-preimports` flag. + """ + with tempfile.TemporaryDirectory() as tmpdir: + temp_dpath = ub.Path(tmpdir) + _write_demo_module(temp_dpath) + + args = [sys.executable, '-m', 'kernprof'] + if prof_mod is not None: + if isinstance(prof_mod, str): + prof_mod = [prof_mod] + for target in prof_mod: + args.extend(['-p', target]) + if eager_preimports in (True,): + args.append('-e') + elif eager_preimports is not None: + if isinstance(eager_preimports, str): + eager_preimports = [eager_preimports] + for target in eager_preimports: + args.extend(['-e', target]) + args.extend(['-l', 'script.py']) + proc = ub.cmd(args, cwd=temp_dpath, verbose=2) + # Check that pre-imports don't accidentally run the code twice + assert proc.stdout.count('7.9') == 1 + print(proc.stdout) + print(proc.stderr) + proc.check_returncode() + + prof = temp_dpath / 'script.py.lprof' + + args = [sys.executable, '-m', 'line_profiler', os.fspath(prof)] + proc = ub.cmd(args, cwd=temp_dpath) + raw_output = proc.stdout + print(raw_output) + proc.check_returncode() + + assert ('Function: add_one' in raw_output) == add_one + assert ('Function: add_two' in raw_output) == add_two + assert ('Function: add_three' in raw_output) == add_three + assert ('Function: add_four' in raw_output) == add_four + assert ('Function: add_operator' in raw_output) == add_operator + assert ('Function: main' in raw_output) == main + + +@pytest.mark.parametrize( + ('eager_preimports, function, method, class_method, static_method, ' + 'descriptor'), + [('my_module', True, True, True, True, True), + # `function()` included in profiling via `Class.partial_method()` + ('my_module.Class', True, True, True, True, True), + ('my_module.Class.descriptor', False, False, False, False, True)]) +def test_autoprofile_callable_wrapper_objects( + eager_preimports, function, method, class_method, static_method, + descriptor): + """ + Test that on-import profiling catches various callable-wrapper + object types: + - properties + - staticmethod + - classmethod + - partialmethod + Like it does regular methods and functions. + """ + with tempfile.TemporaryDirectory() as tmpdir: + temp_dpath = ub.Path(tmpdir) + path = temp_dpath / 'path' + path.mkdir() + (path / 'my_module.py').write_text(ub.codeblock(""" + import functools + + + def function(x): + return + + + class Class: + def method(self): + return + + @classmethod + def class_method(cls): + return + + @staticmethod + def static_method(): + return + + partial_method = functools.partial(function) + + @property + def descriptor(self): + return + """)) + (temp_dpath / 'script.py').write_text(ub.codeblock(""" + import my_module + + + if __name__ == '__main__': + pass + """)) + + with ub.ChDir(temp_dpath): + args = [sys.executable, '-m', 'kernprof', + '-e', eager_preimports, '-lv', 'script.py'] + python_path = os.environ.get('PYTHONPATH') + if python_path: + python_path = '{}:{}'.format(path, python_path) + else: + python_path = str(path) + proc = ub.cmd(args, + env={**os.environ, 'PYTHONPATH': python_path}, + verbose=2) + raw_output = proc.stdout + print(raw_output) + print(proc.stderr) + proc.check_returncode() + + assert ('Function: function' in raw_output) == function + assert ('Function: method' in raw_output) == method + assert ('Function: class_method' in raw_output) == class_method + assert ('Function: static_method' in raw_output) == static_method + # `partial_method()` not included as its own item because it's a + # wrapper around `function()` + assert 'Function: partial_method' not in raw_output + assert ('Function: descriptor' in raw_output) == descriptor From 090309f597f85984812a87877c5d282c00d67164 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 21 Apr 2025 11:13:21 +0200 Subject: [PATCH 08/70] Profiling-target-adding tests tests/test_explicit_profile.py test_profiler_add_methods() New test for the `wrap` argument of the `add_imported_function_or_module()`, `.add_class()`, and `.add_module()` methods --- tests/test_explicit_profile.py | 91 ++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/tests/test_explicit_profile.py b/tests/test_explicit_profile.py index 7108a5d6..28362f84 100644 --- a/tests/test_explicit_profile.py +++ b/tests/test_explicit_profile.py @@ -1,3 +1,4 @@ +import contextlib import os import re import sys @@ -407,6 +408,96 @@ def func4(a): temp_dpath.delete() +@pytest.mark.parametrize('reset_enable_count', [True, False]) +@pytest.mark.parametrize('wrap_class, wrap_module', + [(None, None), (False, True), + (True, False), (True, True)]) +def test_profiler_add_methods(wrap_class, wrap_module, reset_enable_count): + """ + Test the `wrap` argument for the + `line_profiler.autoprofile.autoprofile. + _extend_line_profiler_for_profiling_imports()`, + `LineProfiler.add_class()` and `.add_module()` methods. + """ + def write(path, code): + path.write_text(ub.codeblock(code)) + + script = ub.codeblock(''' + from line_profiler import LineProfiler + from line_profiler.autoprofile.autoprofile import ( + _extend_line_profiler_for_profiling_imports as upgrade_profiler) + + import my_module_1 + from my_module_2 import Class + from my_module_3 import func3 + + + profiler = LineProfiler() + upgrade_profiler(profiler) + profiler.add_imported_function_or_module(my_module_1{}) + profiler.add_imported_function_or_module(Class{}) + profiler.add_imported_function_or_module(func3) + + if {}: + for _ in range(profiler.enable_count): + profiler.disable_by_count() + + # `func1()` should only have timing info if `wrap_module` + my_module_1.func1() + # `method2()` should only have timing info if `wrap_class` + Class.method2() + # `func3()` is profiled but don't see any timing info because it + # isn't wrapped and doesn't auto-`.enable()` before being called + func3() + profiler.print_stats(details=True, summarize=True) + '''.format( + '' if wrap_module is None else f', wrap={wrap_module}', + '' if wrap_class is None else f', wrap={wrap_class}', + reset_enable_count)) + + with contextlib.ExitStack() as stack: + enter = stack.enter_context + enter(ub.ChDir(enter(tempfile.TemporaryDirectory()))) + curdir = ub.Path.cwd() + write(curdir / 'script.py', script) + write(curdir / 'my_module_1.py', + ''' + def func1(): + pass # Marker: func1 + ''') + write(curdir / 'my_module_2.py', + ''' + class Class: + @classmethod + def method2(cls): + pass # Marker: method2 + ''') + write(curdir / 'my_module_3.py', + ''' + def func3(): + pass # Marker: func3 + ''') + proc = ub.cmd([sys.executable, str(curdir / 'script.py')]) + + # Check that the profiler has seen each of the methods + raw_output = proc.stdout + print(script) + print(raw_output) + print(proc.stderr) + proc.check_returncode() + assert '# Marker: func1' in raw_output + assert '# Marker: method2' in raw_output + assert '# Marker: func3' in raw_output + + # Check that the timing info (of the lack thereof) are correct + for func, has_timing in [('func1', wrap_module), ('method2', wrap_class), + ('func3', False)]: + line, = (line for line in raw_output.splitlines() + if line.endswith('Marker: ' + func)) + has_timing = has_timing or not reset_enable_count + assert line.split()[1] == ('1' if has_timing else 'pass') + + if __name__ == '__main__': ... test_simple_explicit_nonglobal_usage() From 7938681311ffbd0923ae877b3ce7426690316380 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 21 Apr 2025 12:03:07 +0200 Subject: [PATCH 09/70] Fixed recursion bug line_profiler/line_profiler.py::LineProfiler.add_class(), .add_module() Added object-identity tracking to avoid possible recursion if the namespaces profiled reference themselves or one another tests/test_explicit_profile.py test_profiler_add_class_recursion_guard() New test for adding self-/mutually-referential classes to be profiled --- line_profiler/line_profiler.py | 12 ++++++-- tests/test_explicit_profile.py | 51 ++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/line_profiler/line_profiler.py b/line_profiler/line_profiler.py index fd1cb8e7..fcaa5322 100755 --- a/line_profiler/line_profiler.py +++ b/line_profiler/line_profiler.py @@ -166,12 +166,20 @@ def _add_namespace(self, namespace, *, wrap=False): n (int): Number of members added to the profiler. """ + return self._add_namespace_inner(set(), namespace, wrap=wrap) + + def _add_namespace_inner(self, duplicate_tracker, namespace, *, + wrap=False): count = 0 - add_cls = self.add_class + add_cls = self._add_namespace_inner add_func = self.add_callable for attr, value in vars(namespace).items(): + if id(value) in duplicate_tracker: + continue + duplicate_tracker.add(id(value)) if isinstance(value, type): - count += 1 if add_cls(value, wrap=wrap) else 0 + if add_cls(duplicate_tracker, value, wrap=wrap): + count += 1 continue try: func_needs_adding = add_func(value) diff --git a/tests/test_explicit_profile.py b/tests/test_explicit_profile.py index 28362f84..a11763b8 100644 --- a/tests/test_explicit_profile.py +++ b/tests/test_explicit_profile.py @@ -498,6 +498,57 @@ def func3(): assert line.split()[1] == ('1' if has_timing else 'pass') +def test_profiler_add_class_recursion_guard(): + """ + Test that if we were to add a pair of classes which each of them + has a reference to the other in its namespace, we don't end up in + infinite recursion. + """ + with contextlib.ExitStack() as stack: + enter = stack.enter_context + enter(ub.ChDir(enter(tempfile.TemporaryDirectory()))) + curdir = ub.Path.cwd() + (curdir / 'script.py').write_text(ub.codeblock(""" + from line_profiler import LineProfiler + + + class Class1: + def method1(self): + pass + + class ChildClass1: + def child_method_1(self): + pass + + + class Class2: + def method2(self): + pass + + class ChildClass2: + def child_method_2(self): + pass + + OtherClass = Class1 + # A duplicate reference shouldn't affect profiling either + YetAnotherClass = Class1 + + + # Add self/mutual references + Class1.ThisClass = Class1 + Class1.OtherClass = Class2 + + profile = LineProfiler() + profile.add_class(Class1) + assert len(profile.functions) == 4 + assert Class1.method1 in profile.functions + assert Class2.method2 in profile.functions + assert Class1.ChildClass1.child_method_1 in profile.functions + assert Class2.ChildClass2.child_method_2 in profile.functions + """)) + ub.cmd([sys.executable, 'script.py'], verbose=2).check_returncode() + + if __name__ == '__main__': ... test_simple_explicit_nonglobal_usage() From 0b0dbb01bb2b62c56c13b00725df35e8a8613acb Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 21 Apr 2025 12:22:18 +0200 Subject: [PATCH 10/70] CHANGELOG entry and doc fix CHANGELOG.rst Added entry for the PR tests/test_explicit_profile.py test_explicit_profile_with_duplicate_functions() Reworded docstring --- CHANGELOG.rst | 1 + tests/test_explicit_profile.py | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0ca0cd22..ec860a70 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -15,6 +15,7 @@ Changes * ENH: In Python >=3.11, profiled objects are reported using their qualified name. * ENH: Highlight final summary using rich if enabled * ENH: Made it possible to use multiple profiler instances simultaneously +* ENH: Added CLI argument ``-e``/``--eager-preimports`` to profile target entities even when they aren't directly imported in the run script/module; made on-import profiling more aggressive so that it doesn't miss entities like class methods and properties 4.2.0 ~~~~~ diff --git a/tests/test_explicit_profile.py b/tests/test_explicit_profile.py index a11763b8..00520a96 100644 --- a/tests/test_explicit_profile.py +++ b/tests/test_explicit_profile.py @@ -415,9 +415,10 @@ def func4(a): def test_profiler_add_methods(wrap_class, wrap_module, reset_enable_count): """ Test the `wrap` argument for the + `LineProfiler.add_class()`, `.add_module()`, and + `.add_imported_function_or_module()` (added via `line_profiler.autoprofile.autoprofile. - _extend_line_profiler_for_profiling_imports()`, - `LineProfiler.add_class()` and `.add_module()` methods. + _extend_line_profiler_for_profiling_imports()`) methods. """ def write(path, code): path.write_text(ub.codeblock(code)) @@ -434,7 +435,9 @@ def write(path, code): profiler = LineProfiler() upgrade_profiler(profiler) + # This dispatches to `.add_module()` profiler.add_imported_function_or_module(my_module_1{}) + # This dispatches to `.add_class()` profiler.add_imported_function_or_module(Class{}) profiler.add_imported_function_or_module(func3) From 388d2dd3ed31170fd34475c728182a7cf5330348 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 21 Apr 2025 13:46:40 +0200 Subject: [PATCH 11/70] Compatibility fix with `pytest < 8` tests/test_eager_preimports.py::create_doctest_wrapper() - Fixed compatibility error (`_pytest.doctest.get_optionflags()` expecting a `pytest.Config` in v8+ but an object having it at `.config` below) - Future-proofed against `pytest` API changes by falling back to the vanilla `doctest` implementation with a warning if setting up the test item fails --- tests/test_eager_preimports.py | 37 +++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/tests/test_eager_preimports.py b/tests/test_eager_preimports.py index b070c63f..9b2a9fc5 100644 --- a/tests/test_eager_preimports.py +++ b/tests/test_eager_preimports.py @@ -11,6 +11,8 @@ import functools import importlib import pathlib +import warnings +import traceback from types import ModuleType from typing import Any, Callable, Dict, Type, Union @@ -75,15 +77,32 @@ def create_doctest_wrapper( use_pytest_doctest = False def wrapper_pytest(request: pytest.FixtureRequest) -> None: - if globs is not None: - test.globs = globs.copy() - module = module_from_parent(parent=request.session, path=fname) - runner = PytestDoctestRunner( - checker=checker, - optionflags=get_doctest_option_flags(request.config), - continue_on_failure=False) - item = item_from_parent(module, name=name, runner=runner, dtest=test) - item.setup() + try: + if globs is not None: + test.globs = globs.copy() + module = module_from_parent(parent=request.session, path=fname) + try: + option_flags = get_doctest_option_flags(request.config) + except AttributeError: + # `pytest < 8` expects an object having the `.config` to + # be passed, instead of the `pytest.Config` itself + option_flags = get_doctest_option_flags(request) + runner = PytestDoctestRunner(checker=checker, + optionflags=option_flags, + continue_on_failure=False) + item = item_from_parent(module, + name=name, runner=runner, dtest=test) + item.setup() + except Exception as e: + # If setting up the test item fails (e.g. due to `pytest` + # refactoring), fall back to the vanilla implementation with + # a warning + msg = ('failed to convert `doctest.DocTest` into ' + '`pytest.Item`:\n\n' + f'{"".join(traceback.format_exception(e))}\n' + 'falling back to vanilla `doctest`') + warnings.warn(msg) + return wrapper_vanilla() try: item.runtest() except doctest.UnexpectedException as e: From d3026efb14bd0d3cc6741a9224a9b1c1290a6a87 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 21 Apr 2025 14:17:31 +0200 Subject: [PATCH 12/70] Fixed `traceback` call (Python < 3.10 compat.) --- tests/test_eager_preimports.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_eager_preimports.py b/tests/test_eager_preimports.py index 9b2a9fc5..ed62039a 100644 --- a/tests/test_eager_preimports.py +++ b/tests/test_eager_preimports.py @@ -97,9 +97,9 @@ def wrapper_pytest(request: pytest.FixtureRequest) -> None: # If setting up the test item fails (e.g. due to `pytest` # refactoring), fall back to the vanilla implementation with # a warning + tb_lines = traceback.format_exception(type(e), e, e.__traceback__) msg = ('failed to convert `doctest.DocTest` into ' - '`pytest.Item`:\n\n' - f'{"".join(traceback.format_exception(e))}\n' + f'`pytest.Item`:\n\n{"".join(tb_lines)}\n' 'falling back to vanilla `doctest`') warnings.warn(msg) return wrapper_vanilla() From 410f9c90a3a5840a85abbf31094d33dafe642783 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Thu, 24 Apr 2025 22:06:05 +0200 Subject: [PATCH 13/70] Scope matching for `LineProfiler.add_*()` line_profiler/autoprofile/line_profiler_utils.py[i] add_imported_function_or_module() Added the `match_scope` argument for limiting the scope of descension into classes in namespaces (classes and modules) line_profiler/line_profiler.py[i] is_c_level_callable() New check for non-profilable C-level callables LineProfiler add_callable(), wrap_callable() Now no-ops on C-level callables add_class(), add_module() - Added the `match_scope` argument for limiting the scope of descension into classes in namespaces (classes and modules) - Added handling for when the `setattr()` on the namespace fails __call__() Added missing method in stub file --- .../autoprofile/line_profiler_utils.py | 22 +- .../autoprofile/line_profiler_utils.pyi | 17 +- line_profiler/line_profiler.py | 196 +++++++++++++++--- line_profiler/line_profiler.pyi | 48 ++++- 4 files changed, 243 insertions(+), 40 deletions(-) diff --git a/line_profiler/autoprofile/line_profiler_utils.py b/line_profiler/autoprofile/line_profiler_utils.py index cbd6e6c2..5867587a 100644 --- a/line_profiler/autoprofile/line_profiler_utils.py +++ b/line_profiler/autoprofile/line_profiler_utils.py @@ -1,7 +1,8 @@ import inspect -def add_imported_function_or_module(self, item, *, wrap=False): +def add_imported_function_or_module(self, item, *, + match_scope='siblings', wrap=False): """ Method to add an object to `LineProfiler` to be profiled. @@ -12,6 +13,21 @@ def add_imported_function_or_module(self, item, *, wrap=False): Args: item (Union[Callable, Type, ModuleType]): Object to be profiled. + match_scope (Literal['exact', 'siblings', 'descendants', + 'none']): + Whether (and how) to match the scope of member classes to + `item` (if a class or module) and decide on whether to add + them: + - 'exact': only add classes defined locally in the body of + `item` + - 'descendants': only add locally-defined classes and + classes defined in submodules or locally-defined class + bodies, and so on. + - 'siblings': only add classes fulfilling 'descendants', + or defined in the same module as `item` (if a class) or in + sibling modules and subpackages to `item` (if a module) + - 'none': don't check scopes and add all classes in the + namespace wrap (bool): Whether to replace the wrapped members with wrappers which automatically enable/disable the profiler when called. @@ -23,9 +39,9 @@ def add_imported_function_or_module(self, item, *, wrap=False): `LineProfiler.add_callable()`, `.add_module()`, `.add_class()` """ if inspect.isclass(item): - count = self.add_class(item, wrap=wrap) + count = self.add_class(item, match_scope=match_scope, wrap=wrap) elif inspect.ismodule(item): - count = self.add_module(item, wrap=wrap) + count = self.add_module(item, match_scope=match_scope, wrap=wrap) else: try: count = self.add_callable(item) diff --git a/line_profiler/autoprofile/line_profiler_utils.pyi b/line_profiler/autoprofile/line_profiler_utils.pyi index 11d380bc..cfbef479 100644 --- a/line_profiler/autoprofile/line_profiler_utils.pyi +++ b/line_profiler/autoprofile/line_profiler_utils.pyi @@ -1,8 +1,21 @@ -from typing import Callable, Literal from types import ModuleType +from typing import overload, Any, Literal, TYPE_CHECKING +if TYPE_CHECKING: # Stub-only annotations + from ..line_profiler import CLevelCallable, CallableLike, MatchScopeOption + +@overload +def add_imported_function_or_module( + self, item: CLevelCallable | Any, + match_scope: MatchScopeOption = 'siblings', + wrap: bool = False) -> Literal[0]: + ... + + +@overload def add_imported_function_or_module( - self, item: Callable | type | ModuleType, *, + self, item: CallableLike | type | ModuleType, + match_scope: MatchScopeOption = 'siblings', wrap: bool = False) -> Literal[0, 1]: ... diff --git a/line_profiler/line_profiler.py b/line_profiler/line_profiler.py index fcaa5322..96798efe 100755 --- a/line_profiler/line_profiler.py +++ b/line_profiler/line_profiler.py @@ -10,6 +10,8 @@ import pickle import sys import tempfile +import types +import warnings from argparse import ArgumentError, ArgumentParser try: @@ -28,9 +30,28 @@ # NOTE: This needs to be in sync with ../kernprof.py and __init__.py __version__ = '4.3.0' +# These objects are callables, but are defined in C so we can't handle +# them anyway +c_level_callable_types = (types.BuiltinFunctionType, + types.BuiltinMethodType, + types.ClassMethodDescriptorType, + types.MethodDescriptorType, + types.MethodWrapperType, + types.WrapperDescriptorType) + is_function = inspect.isfunction +def is_c_level_callable(func): + """ + Returns: + func_is_c_level (bool): + Whether a callable is defined at the C level (and is thus + non-profilable). + """ + return isinstance(func, c_level_callable_types) + + def load_ipython_extension(ip): """ API for IPython to recognize this module as an IPython extension. """ @@ -63,6 +84,8 @@ def _get_underlying_functions(func): f'cannot get functions from {type(func)} objects') if is_function(func): return [func] + if is_c_level_callable(func): + return [] return [type(func).__call__] @@ -104,12 +127,20 @@ def __call__(self, func): start the profiler on function entry and stop it on function exit. """ - # Note: if `func` is a `types.FunctionType` which is already - # decorated by the profiler, the same object is returned; + # The same object is returned when: + # - `func` is a `types.FunctionType` which is already + # decorated by the profiler, or + # - `func` is any of the C-level callables that can't be + # profiled # otherwise, wrapper objects are always returned. self.add_callable(func) return self.wrap_callable(func) + def wrap_callable(self, func): + if is_c_level_callable(func): # Non-profilable + return func + return super().wrap_callable(func) + def add_callable(self, func): """ Register a function, method, property, partial object, etc. with @@ -149,50 +180,153 @@ def print_stats(self, stream=None, output_unit=None, stripzeros=False, stream=stream, stripzeros=stripzeros, details=details, summarize=summarize, sort=sort, rich=rich) - def _add_namespace(self, namespace, *, wrap=False): - """ - Add the members (callables (wrappers), methods, classes, ...) in - a namespace and profile them. - - Args: - namespace (Union[ModuleType, type]): - Module or class to be profiled. - wrap (bool): - Whether to replace the wrapped members with wrappers - which automatically enable/disable the profiler when - called. - - Returns: - n (int): - Number of members added to the profiler. - """ - return self._add_namespace_inner(set(), namespace, wrap=wrap) - - def _add_namespace_inner(self, duplicate_tracker, namespace, *, - wrap=False): + def _add_namespace(self, duplicate_tracker, namespace, *, + filter_scope=None, wrap=False): count = 0 - add_cls = self._add_namespace_inner + add_cls = self._add_namespace add_func = self.add_callable + wrap_failures = {} + if filter_scope is None: + def filter_scope(*_): + return True + for attr, value in vars(namespace).items(): if id(value) in duplicate_tracker: continue duplicate_tracker.add(id(value)) if isinstance(value, type): - if add_cls(duplicate_tracker, value, wrap=wrap): - count += 1 + if filter_scope(namespace, value): + if add_cls(duplicate_tracker, value, wrap=wrap): + count += 1 continue try: - func_needs_adding = add_func(value) + if not add_func(value): + continue except TypeError: # Not a callable (wrapper) continue - if not func_needs_adding: - continue if wrap: - setattr(namespace, attr, self.wrap_callable(value)) + wrapper = self.wrap_callable(value) + if wrapper is not value: + try: + setattr(namespace, attr, wrapper) + except (TypeError, AttributeError): + # Corner case in case if a class/module don't + # allow setting attributes (could e.g. happen + # with some builtin/extension classes, but their + # method should be in C anyway, so + # `.add_callable()` should've returned 0 and we + # shouldn't be here) + wrap_failures[attr] = value count += 1 + if wrap_failures: + msg = (f'cannot wrap {len(wrap_failures)} attribute(s) of ' + f'{namespace!r} (`{{attr: value}}`): {wrap_failures!r}') + warnings.warn(msg, stacklevel=2) return count - add_class = add_module = _add_namespace + def add_class(self, cls, *, match_scope='siblings', wrap=False): + """ + Add the members (callables (wrappers), methods, classes, ...) in + a class' local namespace and profile them. + + Args: + cls (type): + Class to be profiled. + match_scope (Literal['exact', 'siblings', 'descendants', + 'none']): + Whether (and how) to match the scope of member classes + and decide on whether to add them: + - 'exact': only add classes defined locally in this + namespace, i.e. in the body of `cls`, as "inner + classes" + - 'descendants': only add "inner classes", their "inner + classes", and so on. + - 'siblings': only add classes fulfilling 'descendants', + or defined in the same module as `cls` + - 'none': don't check scopes and add all classes in the + namespace + wrap (bool): + Whether to replace the wrapped members with wrappers + which automatically enable/disable the profiler when + called. + + Returns: + n (int): + Number of members added to the profiler. + """ + def class_is_child(cls, other): + if not modules_are_equal(cls, other): + return False + return other.__qualname__ == f'{cls.__qualname__}.{other.__name__}' + + def modules_are_equal(cls, other): # = sibling check + return cls.__module__ == other.__module__ + + def class_is_descendant(cls, other): + if not modules_are_equal(cls, other): + return False + return other.__qualname__.startswith(cls.__qualname__ + '.') + + filter_scope = {'exact': class_is_child, + 'descendants': class_is_descendant, + 'siblings': modules_are_equal, + 'none': None}[match_scope] + return self._add_namespace(set(), cls, + filter_scope=filter_scope, wrap=wrap) + + def add_module(self, mod, *, match_scope='siblings', wrap=False): + """ + Add the members (callables (wrappers), methods, classes, ...) in + a module's local namespace and profile them. + + Args: + mod (ModuleType): + Module to be profiled. + match_scope (Literal['exact', 'siblings', 'descendants', + 'none']): + Whether (and how) to match the scope of member classes + and decide on whether to add them: + - 'exact': only add classes defined locally in this + namespace, i.e. in the body of `mod` + - 'descendants': only add locally-defined classes, + classes locally defined in their bodies, and so on + - 'siblings': only add classes fulfilling 'descendants', + or defined in sibling modules/subpackages to `mod` (if + `mod` is part of a package) + - 'none': don't check scopes and add all classes in the + namespace + wrap (bool): + Whether to replace the wrapped members with wrappers + which automatically enable/disable the profiler when + called. + + Returns: + n (int): + Number of members added to the profiler. + """ + def match_prefix(s: str, prefix: str, sep: str = '.') -> bool: + return s == prefix or s.startswith(prefix + sep) + + def class_is_child(mod, other): + return other.__module__ == mod.__name__ + + def class_is_descendant(mod, other): + return match_prefix(other.__module__, mod.__name__) + + def class_is_cousin(mod, other): + if class_is_descendant(mod, other): + return True + return match_prefix(other.__module__, parent) + + parent, _, basename = mod.__name__.rpartition('.') + filter_scope = {'exact': class_is_child, + 'descendants': class_is_descendant, + 'siblings': (class_is_cousin # Only if a pkg + if basename else + class_is_descendant), + 'none': None}[match_scope] + return self._add_namespace(set(), mod, + filter_scope=filter_scope, wrap=wrap) def _get_wrapper_info(self, func): info = getattr(func, self._profiler_wrapped_marker, None) diff --git a/line_profiler/line_profiler.pyi b/line_profiler/line_profiler.pyi index 1607e5d5..7d4c70b1 100644 --- a/line_profiler/line_profiler.pyi +++ b/line_profiler/line_profiler.pyi @@ -1,16 +1,52 @@ -from types import ModuleType -from typing import Literal, List, Tuple +from functools import cached_property, partial, partialmethod +from inspect import isfunction as is_function +from types import (FunctionType, MethodType, ModuleType, + BuiltinFunctionType, BuiltinMethodType, + ClassMethodDescriptorType, MethodDescriptorType, + MethodWrapperType, WrapperDescriptorType) +from typing import (overload, + Any, Literal, Callable, List, Tuple, TypeGuard, TypeVar) import io from ._line_profiler import LineProfiler as CLineProfiler from .profiler_mixin import ByCountProfilerMixin from _typeshed import Incomplete +CLevelCallable = TypeVar('CLevelCallable', + BuiltinFunctionType, BuiltinMethodType, + ClassMethodDescriptorType, MethodDescriptorType, + MethodWrapperType, WrapperDescriptorType) +CallableLike = TypeVar('CallableLike', + FunctionType, partial, property, cached_property, + MethodType, staticmethod, classmethod, partialmethod) +MatchScopeOption = Literal['exact', 'descendants', 'siblings', 'none'] + + +def is_c_level_callable(func: Any) -> TypeGuard[CLevelCallable]: + ... + + def load_ipython_extension(ip) -> None: ... class LineProfiler(CLineProfiler, ByCountProfilerMixin): + @overload + def __call__(self, # type: ignore[overload-overlap] + func: CLevelCallable) -> CLevelCallable: + ... + + @overload + def __call__(self, # type: ignore[overload-overlap] + func: CallableLike) -> CallableLike: + ... + + # Fallback: just wrap the `.__call__()` of a generic callable + + @overload + def __call__(self, func: Callable) -> FunctionType: + ... + def add_callable(self, func) -> Literal[0, 1]: ... @@ -27,10 +63,14 @@ class LineProfiler(CLineProfiler, ByCountProfilerMixin): rich: bool = ...) -> None: ... - def add_module(self, mod: ModuleType, *, wrap: bool = False) -> int: + def add_module(self, mod: ModuleType, *, + match_scope: MatchScopeOption = 'siblings', + wrap: bool = False) -> int: ... - def add_class(self, cls: type, *, wrap: bool = False) -> int: + def add_class(self, cls: type, *, + match_scope: MatchScopeOption = 'siblings', + wrap: bool = False) -> int: ... From ab2a716523c8477568cd819d9b5762eacc747d0a Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 27 Apr 2025 03:16:32 +0200 Subject: [PATCH 14/70] Tests and fixes line_profiler/line_profiler.py::LineProfiler.add_class(), add_module() Fixed implementation of internal method used (wrong interpretation of the `match_scope` parameter) tests/test_explicit_profile.py test_profiler_add_methods() test_profiler_add_class_recursion_guard() Simplified implementations test_profiler_warn_unwrappable() New test for the warning in `LineProfiler.add_*(wrap=True)` test_profiler_scope_matching() New test for `LineProfiler.add_*(match_scope=...)` tests/test_line_profiler.py::test_profiler_c_callable_no_op() New test for how the profiler leaves C-level callables untouched --- line_profiler/line_profiler.py | 109 ++++++++++--------- tests/test_explicit_profile.py | 193 ++++++++++++++++++++++++++++----- tests/test_line_profiler.py | 22 ++++ 3 files changed, 247 insertions(+), 77 deletions(-) diff --git a/line_profiler/line_profiler.py b/line_profiler/line_profiler.py index 96798efe..7c0c79bc 100755 --- a/line_profiler/line_profiler.py +++ b/line_profiler/line_profiler.py @@ -4,6 +4,7 @@ inspect its output. This depends on the :py:mod:`line_profiler._line_profiler` Cython backend. """ +import functools import inspect import linecache import os @@ -181,23 +182,29 @@ def print_stats(self, stream=None, output_unit=None, stripzeros=False, details=details, summarize=summarize, sort=sort, rich=rich) def _add_namespace(self, duplicate_tracker, namespace, *, - filter_scope=None, wrap=False): + match_scope='none', wrap=False): count = 0 - add_cls = self._add_namespace + add_cls = functools.partial(self._add_namespace, duplicate_tracker, + match_scope=match_scope, wrap=wrap) add_func = self.add_callable + remember = duplicate_tracker.add + wrap_func = self.wrap_callable wrap_failures = {} - if filter_scope is None: - def filter_scope(*_): + if match_scope == 'none': + def check(*_): return True + elif isinstance(namespace, type): + check = self._add_class_filter(namespace, match_scope) + else: + check = self._add_module_filter(namespace, match_scope) for attr, value in vars(namespace).items(): if id(value) in duplicate_tracker: continue - duplicate_tracker.add(id(value)) + remember(id(value)) if isinstance(value, type): - if filter_scope(namespace, value): - if add_cls(duplicate_tracker, value, wrap=wrap): - count += 1 + if check(value) and add_cls(value): + count += 1 continue try: if not add_func(value): @@ -205,7 +212,7 @@ def filter_scope(*_): except TypeError: # Not a callable (wrapper) continue if wrap: - wrapper = self.wrap_callable(value) + wrapper = add_func(value) if wrapper is not value: try: setattr(namespace, attr, wrapper) @@ -224,6 +231,48 @@ def filter_scope(*_): warnings.warn(msg, stacklevel=2) return count + @staticmethod + def _add_module_filter(mod, match_scope): + def match_prefix(s, prefix, sep='.'): + return s == prefix or s.startswith(prefix + sep) + + def class_is_child(other): + return other.__module__ == mod.__name__ + + def class_is_descendant(other): + return match_prefix(other.__module__, mod.__name__) + + def class_is_cousin(other): + if class_is_descendant(other): + return True + return match_prefix(other.__module__, parent) + + parent, _, basename = mod.__name__.rpartition('.') + return {'exact': class_is_child, + 'descendants': class_is_descendant, + 'siblings': (class_is_cousin # Only if a pkg + if basename else + class_is_descendant)}[match_scope] + + @staticmethod + def _add_class_filter(cls, match_scope): + def class_is_child(other): + if not modules_are_equal(other): + return False + return other.__qualname__ == f'{cls.__qualname__}.{other.__name__}' + + def modules_are_equal(other): # = sibling check + return cls.__module__ == other.__module__ + + def class_is_descendant(other): + if not modules_are_equal(other): + return False + return other.__qualname__.startswith(cls.__qualname__ + '.') + + return {'exact': class_is_child, + 'descendants': class_is_descendant, + 'siblings': modules_are_equal}[match_scope] + def add_class(self, cls, *, match_scope='siblings', wrap=False): """ Add the members (callables (wrappers), methods, classes, ...) in @@ -254,25 +303,8 @@ def add_class(self, cls, *, match_scope='siblings', wrap=False): n (int): Number of members added to the profiler. """ - def class_is_child(cls, other): - if not modules_are_equal(cls, other): - return False - return other.__qualname__ == f'{cls.__qualname__}.{other.__name__}' - - def modules_are_equal(cls, other): # = sibling check - return cls.__module__ == other.__module__ - - def class_is_descendant(cls, other): - if not modules_are_equal(cls, other): - return False - return other.__qualname__.startswith(cls.__qualname__ + '.') - - filter_scope = {'exact': class_is_child, - 'descendants': class_is_descendant, - 'siblings': modules_are_equal, - 'none': None}[match_scope] return self._add_namespace(set(), cls, - filter_scope=filter_scope, wrap=wrap) + match_scope=match_scope, wrap=wrap) def add_module(self, mod, *, match_scope='siblings', wrap=False): """ @@ -304,29 +336,8 @@ def add_module(self, mod, *, match_scope='siblings', wrap=False): n (int): Number of members added to the profiler. """ - def match_prefix(s: str, prefix: str, sep: str = '.') -> bool: - return s == prefix or s.startswith(prefix + sep) - - def class_is_child(mod, other): - return other.__module__ == mod.__name__ - - def class_is_descendant(mod, other): - return match_prefix(other.__module__, mod.__name__) - - def class_is_cousin(mod, other): - if class_is_descendant(mod, other): - return True - return match_prefix(other.__module__, parent) - - parent, _, basename = mod.__name__.rpartition('.') - filter_scope = {'exact': class_is_child, - 'descendants': class_is_descendant, - 'siblings': (class_is_cousin # Only if a pkg - if basename else - class_is_descendant), - 'none': None}[match_scope] return self._add_namespace(set(), mod, - filter_scope=filter_scope, wrap=wrap) + match_scope=match_scope, wrap=wrap) def _get_wrapper_info(self, func): info = getattr(func, self._profiler_wrapped_marker, None) diff --git a/tests/test_explicit_profile.py b/tests/test_explicit_profile.py index 00520a96..640d7549 100644 --- a/tests/test_explicit_profile.py +++ b/tests/test_explicit_profile.py @@ -8,6 +8,15 @@ import ubelt as ub +@contextlib.contextmanager +def enter_tmpdir(): + with contextlib.ExitStack() as stack: + enter = stack.enter_context + tmpdir = os.path.abspath(enter(tempfile.TemporaryDirectory())) + enter(ub.ChDir(tmpdir)) + yield ub.Path(tmpdir) + + def test_simple_explicit_nonglobal_usage(): """ python -c "from test_explicit_profile import *; test_simple_explicit_nonglobal_usage()" @@ -458,10 +467,7 @@ def write(path, code): '' if wrap_class is None else f', wrap={wrap_class}', reset_enable_count)) - with contextlib.ExitStack() as stack: - enter = stack.enter_context - enter(ub.ChDir(enter(tempfile.TemporaryDirectory()))) - curdir = ub.Path.cwd() + with enter_tmpdir() as curdir: write(curdir / 'script.py', script) write(curdir / 'my_module_1.py', ''' @@ -507,49 +513,180 @@ def test_profiler_add_class_recursion_guard(): has a reference to the other in its namespace, we don't end up in infinite recursion. """ - with contextlib.ExitStack() as stack: - enter = stack.enter_context - enter(ub.ChDir(enter(tempfile.TemporaryDirectory()))) - curdir = ub.Path.cwd() - (curdir / 'script.py').write_text(ub.codeblock(""" - from line_profiler import LineProfiler + from line_profiler import LineProfiler + + class Class1: + def method1(self): + pass + + class ChildClass1: + def child_method_1(self): + pass + + class Class2: + def method2(self): + pass + + class ChildClass2: + def child_method_2(self): + pass + + OtherClass = Class1 + # A duplicate reference shouldn't affect profiling either + YetAnotherClass = Class1 + + # Add self/mutual references + Class1.ThisClass = Class1 + Class1.OtherClass = Class2 + profile = LineProfiler() + profile.add_class(Class1) + assert len(profile.functions) == 4 + assert Class1.method1 in profile.functions + assert Class2.method2 in profile.functions + assert Class1.ChildClass1.child_method_1 in profile.functions + assert Class2.ChildClass2.child_method_2 in profile.functions + +def test_profiler_warn_unwrappable(): + """ + Test for warnings when using `LineProfiler.add_*(wrap=True)` with a + namespace which doesn't allow attribute assignment. + """ + from line_profiler import LineProfiler + + class ProblamticMeta(type): + def __init__(cls, *args, **kwargs): + super(ProblamticMeta, cls).__init__(*args, **kwargs) + cls._initialized = True + + def __setattr__(cls, attr, value): + if not getattr(cls, '_initialized', None): + return super(ProblamticMeta, cls).__setattr__(attr, value) + raise AttributeError( + f'cannot set attribute on {type(cls)} instance') + + class ProblematicClass(metaclass=ProblamticMeta): + def method(self): + pass + + profile = LineProfiler() + vanilla_method = ProblematicClass.method + + with pytest.warns(match=r"cannot wrap 1 attribute\(s\) of " + r" \(`\{attr: value\}`\): " + r"\{'method': \}"): + # The method is added to the profiler, but we can't assign its + # wrapper back into the class namespace + assert profile.add_class(ProblematicClass, wrap=True) == 1 + + assert ProblematicClass.method is vanilla_method + + +@pytest.mark.parametrize( + ('match_scope', 'add_module_targets', 'add_class_targets'), + [('exact', + {'class2_method', 'child_class2_method'}, + {'class3_method', 'child_class3_method'}), + ('descendants', + {'class2_method', 'child_class2_method', + 'class3_method', 'child_class3_method'}, + {'class3_method', 'child_class3_method'}), + ('siblings', + {'class1_method', 'child_class1_method', + 'class2_method', 'child_class2_method', + 'class3_method', 'child_class3_method', 'other_class3_method'}, + {'class3_method', 'child_class3_method', 'other_class3_method'}), + ('none', + {'class1_method', 'child_class1_method', + 'class2_method', 'child_class2_method', + 'class3_method', 'child_class3_method', 'other_class3_method'}, + {'child_class1_method', + 'class3_method', 'child_class3_method', 'other_class3_method'})]) +def test_profiler_scope_matching(monkeypatch, + match_scope, + add_module_targets, + add_class_targets): + """ + Test for the scope-matching strategies of the `LineProfiler.add_*()` + methods. + """ + def write(path, code=None): + path.parent.mkdir(exist_ok=True, parents=True) + if code is None: + path.touch() + else: + path.write_text(ub.codeblock(code)) + + with enter_tmpdir() as curdir: + pkg_dir = curdir / 'packages' / 'my_pkg' + write(pkg_dir / '__init__.py') + write(pkg_dir / 'submod1.py', + """ class Class1: - def method1(self): + def class1_method(self): pass class ChildClass1: - def child_method_1(self): + def child_class1_method(self): pass + """) + write(pkg_dir / 'subpkg2' / '__init__.py', + """ + from ..submod1 import Class1 # Import from a sibling + from .submod3 import Class3 # Import from a descendant class Class2: - def method2(self): + def class2_method(self): pass class ChildClass2: - def child_method_2(self): + def child_class2_method(self): pass - OtherClass = Class1 - # A duplicate reference shouldn't affect profiling either - YetAnotherClass = Class1 + BorrowedChildClass = Class1.ChildClass1 # Non-sibling class + """) + write(pkg_dir / 'subpkg2' / 'submod3.py', + """ + from ..submod1 import Class1 - # Add self/mutual references - Class1.ThisClass = Class1 - Class1.OtherClass = Class2 + class Class3: + def class3_method(self): + pass + class OtherChildClass3: + def child_class3_method(self): + pass + + # Unrelated class + BorrowedChildClass1 = Class1.ChildClass1 + + class OtherClass3: + def other_class3_method(self): + pass + + # Sibling class + Class3.BorrowedChildClass3 = OtherClass3 + """) + monkeypatch.syspath_prepend(pkg_dir.parent) + + from my_pkg import subpkg2 + from line_profiler import LineProfiler + + # Add a module + profile = LineProfiler() + profile.add_module(subpkg2, match_scope=match_scope) + assert len(profile.functions) == len(add_module_targets) + added = {func.__name__ for func in profile.functions} + assert added == set(add_module_targets) + # Add a class profile = LineProfiler() - profile.add_class(Class1) - assert len(profile.functions) == 4 - assert Class1.method1 in profile.functions - assert Class2.method2 in profile.functions - assert Class1.ChildClass1.child_method_1 in profile.functions - assert Class2.ChildClass2.child_method_2 in profile.functions - """)) - ub.cmd([sys.executable, 'script.py'], verbose=2).check_returncode() + profile.add_class(subpkg2.Class3, match_scope=match_scope) + assert len(profile.functions) == len(add_class_targets) + added = {func.__name__ for func in profile.functions} + assert added == set(add_class_targets) if __name__ == '__main__': diff --git a/tests/test_line_profiler.py b/tests/test_line_profiler.py index c922109a..563e1a19 100644 --- a/tests/test_line_profiler.py +++ b/tests/test_line_profiler.py @@ -506,6 +506,28 @@ def foo(self) -> int: assert profile.enable_count == 0 +def test_profiler_c_callable_no_op(): + """ + Test that when the profiler is used to decorate or add a C-level + callable it results in a no-op. + """ + profile = LineProfiler() + + for i, (func, Type) in enumerate([ + (len, types.BuiltinFunctionType), + ('string'.split, types.BuiltinMethodType), + (vars(int)['from_bytes'], types.ClassMethodDescriptorType), + (str.split, types.MethodDescriptorType), + ((1).__str__, types.MethodWrapperType), + (int.__repr__, types.WrapperDescriptorType)]): + assert isinstance(func, Type) + if i % 2: # Add is no-op + assert not profile.add_callable(func) + else: # Decoration is no-op + assert profile(func) is func + assert not profile.functions + + def test_show_func_column_formatting(): from line_profiler.line_profiler import show_func import line_profiler From a2bf8457b6865cc37854e1db5513346c3e511d25 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 27 Apr 2025 04:02:01 +0200 Subject: [PATCH 15/70] Fixed typo --- line_profiler/line_profiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/line_profiler/line_profiler.py b/line_profiler/line_profiler.py index 7c0c79bc..cba36364 100755 --- a/line_profiler/line_profiler.py +++ b/line_profiler/line_profiler.py @@ -212,7 +212,7 @@ def check(*_): except TypeError: # Not a callable (wrapper) continue if wrap: - wrapper = add_func(value) + wrapper = wrap_func(value) if wrapper is not value: try: setattr(namespace, attr, wrapper) From 635f49fe7466428a507d2d4299c627e4794893c4 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 18 May 2025 05:58:13 +0200 Subject: [PATCH 16/70] Refactoring: default to eager imports CHANGERLOG.rst - Reworded previous entry (#338) - Reworded entry kernprof.py __doc__ Updated _normalize_profiling_targets.__doc__ _restore_list.__doc__ pre_parse_single_arg_directive.__doc__ Reformatted to be more `sphinx`-friendly main() - Removed the `-e`/`--eager-preimports` flag - Made eager pre-imports the default for `--prof-mod` - Added new flag `--no-preimports` for restoring the old behavior --- CHANGELOG.rst | 4 +-- kernprof.py | 97 +++++++++++++++++++++++---------------------------- 2 files changed, 46 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ec860a70..6b96a08e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,11 +11,11 @@ Changes * FIX: Fixed explicit profiling of class methods; added handling for profiling static, bound, and partial methods, ``functools.partial`` objects, (cached) properties, and async generator functions * FIX: Fixed namespace bug when running ``kernprof -m`` on certain modules (e.g. ``calendar`` on Python 3.12+). * FIX: Fixed ``@contextlib.contextmanager`` bug where the cleanup code (e.g. restoration of ``sys`` attributes) is not run if exceptions occurred inside the context -* ENH: Added CLI arguments ``-c`` to ``kernprof`` for (auto-)profiling module/package/inline-script execution instead of that of script files; passing ``'-'`` as the script-file name now also reads from and profiles ``stdin`` +* ENH: Added CLI arguments ``-c`` to ``kernprof`` for (auto-)profiling inline-script execution instead of that of script files; passing ``'-'`` as the script-file name now also reads from and profiles ``stdin`` * ENH: In Python >=3.11, profiled objects are reported using their qualified name. * ENH: Highlight final summary using rich if enabled * ENH: Made it possible to use multiple profiler instances simultaneously -* ENH: Added CLI argument ``-e``/``--eager-preimports`` to profile target entities even when they aren't directly imported in the run script/module; made on-import profiling more aggressive so that it doesn't miss entities like class methods and properties +* ENH: ``kernprof --prof-mod`` target entities are now imported and profiled regardless of whether they are directly imported in the run script/module/code (old behavior recoed by passing ``--no-preimports``); made on-import profiling more aggressive so that it doesn't miss entities like class methods and properties 4.2.0 ~~~~~ diff --git a/kernprof.py b/kernprof.py index 8518d197..f407f537 100755 --- a/kernprof.py +++ b/kernprof.py @@ -41,7 +41,7 @@ def main(): NOTE: - New in 4.3.0: more code execution options are added: + New in 4.3.0: More code execution options are added: * ``kernprof -m some.module `` parallels ``python -m`` and runs the provided module as ``__main__``. @@ -88,15 +88,22 @@ def main(): -i, --output-interval [OUTPUT_INTERVAL] Enables outputting of cumulative profiling results to file every n seconds. Uses the threading module. Minimum value is 1 (second). Defaults to disabled. - -p, --prof-mod PROF_MOD - List of modules, functions and/or classes to profile specified by their name or path, if they are imported in the profiled script/module. These - profiling targets can be supplied both as comma-separated items, or separately with multiple copies of this flag. Adding the current script/module - profiles the entirety of it. Only works with line_profiler -l, --line-by-line. - -e, --eager-preimports - List of modules, functions, and/or classes, to be imported and marked for profiling before running the script/module, regardless of whether they are - directly imported in the script/module. Follows the same semantics as `--prof-mod`. If supplied without an argument, indicates that all `--prof-mod` - targets are to be so profiled. Only works with line_profiler -l, --line-by-line. + -p, --prof-mod {path/to/script | object.dotted.path}[,...] + List of modules, functions and/or classes to profile specified by their name or path. These profiling targets can be supplied both as comma-separated + items, or separately with multiple copies of this flag. Adding the current script/module profiles the entirety of it. Only works with line_profiler -l, + --no-preimports Instead of eagerly importing all profiling targets specified via -p and profiling them, only profile those that are directly imported in the profiled + code. Only works with line_profiler -l, --line-by-line. --prof-imports If specified, modules specified to `--prof-mod` will also autoprofile modules that they import. Only works with line_profiler -l, --line-by-line + +NOTE: + + New in 4.3.0: For more intuitive profiling behavior, profiling + targets in ``--prof-mod`` (except the profiled script/code) are now + eagerly pre-imported to be profiled + (see :py:mod:`line_profiler.autoprofile.eager_preimports`), + regardless of whether those imports directly occur in the profiled + script/module/code. + To restore the old behavior, pass the ``--no-preimports`` flag. """ import builtins import functools @@ -249,12 +256,13 @@ def _python_command(): def _normalize_profiling_targets(targets): """ - Normalize the parsed `--prof-mod` and `--eager-preimports` by: - - Normalizing file paths with `find_script()`, and subsequently - to absolute paths. - - Splitting non-file paths at commas into (presumably) file paths + Normalize the parsed ``--prof-mod`` by: + + * Normalizing file paths with :py:func:`find_script()`, and + subsequently to absolute paths. + * Splitting non-file paths at commas into (presumably) file paths and/or dotted paths. - - Removing duplicates. + * Removing duplicates. """ def find(path): try: @@ -277,8 +285,8 @@ def find(path): class _restore_list: """ - Restore a list like `sys.path` after running code which potentially - modifies it. + Restore a list like ``sys.path`` after running code which + potentially modifies it. Example ------- @@ -320,8 +328,8 @@ def wrapper(*args, **kwargs): def pre_parse_single_arg_directive(args, flag, sep='--'): """ - Pre-parse high-priority single-argument directives like `-m module` - to emulate the behavior of `python [...]`. + Pre-parse high-priority single-argument directives like + ``-m module`` to emulate the behavior of ``python [...]``. Examples -------- @@ -455,28 +463,19 @@ def positive_float(value): metavar=("{path/to/script | object.dotted.path}" "[,...]"), help="List of modules, functions and/or classes " - "to profile specified by their name or path, " - "if they are imported in the profiled " - "script/module. " + "to profile specified by their name or path. " "These profiling targets can be supplied both as " "comma-separated items, or separately with " "multiple copies of this flag. " "Adding the current script/module profiles the " "entirety of it. " "Only works with line_profiler -l, --line-by-line.") - parser.add_argument('-e', '--eager-preimports', - action='append', - const=True, - metavar=("{path/to/script | object.dotted.path}" - "[,...]"), - nargs='?', - help="List of modules, functions, and/or classes, " - "to be imported and marked for profiling before " - "running the script/module, regardless of whether " - "they are directly imported in the script/module. " - "Follows the same semantics as `--prof-mod`. " - "If supplied without an argument, indicates that " - "all `--prof-mod` targets are to be so profiled. " + parser.add_argument('--no-preimports', + action='store_true', + help="Instead of eagerly importing all profiling " + "targets specified via -p and profiling them, " + "only profile those that are directly imported in " + "the profiled code. " "Only works with line_profiler -l, --line-by-line.") parser.add_argument('--prof-imports', action='store_true', help="If specified, modules specified to `--prof-mod` will also autoprofile modules that they import. " @@ -619,20 +618,14 @@ def _main(options, module=False): # commas), so check against existing filenames before splitting # them options.prof_mod = _normalize_profiling_targets(options.prof_mod) - if options.eager_preimports: - if options.eager_preimports == [True]: - # Eager-import all of `--prof-mod` - options.eager_preimports = list(options.prof_mod or []) - else: # Only eager-import the specified targets - options.eager_preimports = _normalize_profiling_targets([ - target for target in options.eager_preimports - if target not in (True,)]) - if options.line_by_line and options.eager_preimports: - # We assume most items in `.eager_preimports` to be import-able - # without significant side effects, but the same cannot be said - # if it contains the script file to be run. E.g. the script may - # not even have a `if __name__ == '__main__': ...` guard. So - # don't eager-import it. + if not options.prof_mod: + options.no_preimports = True + if options.line_by_line and not options.no_preimports: + # We assume most items in `.prof_mod` to be import-able without + # significant side effects, but the same cannot be said if it + # contains the script file to be run. E.g. the script may not + # even have a `if __name__ == '__main__': ...` guard. So don't + # eager-import it. from line_profiler.autoprofile.eager_preimports import ( is_dotted_path, propose_names, write_eager_import_module) from line_profiler.autoprofile.util_static import modpath_to_modname @@ -641,7 +634,7 @@ def _main(options, module=False): filtered_targets = [] invalid_targets = [] - for target in options.eager_preimports: + for target in options.prof_mod: if is_dotted_path(target): filtered_targets.append(target) continue @@ -652,10 +645,8 @@ def _main(options, module=False): continue if not module and os.path.samefile(target, script_file): # Ignore the script to be run in eager importing - # (but make sure that it is handled by `--prof-mod`) - if options.prof_mod is None: - options.prof_mod = [] - options.prof_mod.append(script_file) + # (`line_profiler.autoprofile.autoprofile.run()` will + # handle it) continue modname = modpath_to_modname(target) if modname is None: # Not import-able From cc77994c3cf825344962c64d4db04222509cf9fd Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 18 May 2025 07:02:01 +0200 Subject: [PATCH 17/70] Updated tests tests/test_autoprofile.py test_autoprofile_exec_package(), test_autoprofile_exec_module() Updated to reflect the new behavior of `kernprof` and to test the `--no-preimports` flag test_autoprofile_eager_preimports() Removed because the `--eager-preimports` flag is removed test_autoprofile_callable_wrapper_objects() Updated because the `--eager-preimports` flag is removed --- tests/test_autoprofile.py | 130 ++++++++++++++------------------------ 1 file changed, 46 insertions(+), 84 deletions(-) diff --git a/tests/test_autoprofile.py b/tests/test_autoprofile.py index 94ed7b14..0aee7354 100644 --- a/tests/test_autoprofile.py +++ b/tests/test_autoprofile.py @@ -432,19 +432,28 @@ def test_autoprofile_script_with_prof_imports(): @pytest.mark.parametrize( - ['use_kernprof_exec', 'prof_mod', 'prof_imports', + ['use_kernprof_exec', 'prof_mod', 'no_preimports', 'prof_imports', 'add_one', 'add_two', 'add_operator', 'main'], - [(False, 'test_mod.submod1', False, True, False, False, False), - (False, 'test_mod.submod2', True, False, True, True, False), - (False, 'test_mod', True, True, True, True, True), + [(False, 'test_mod.submod1', False, False, + True, False, True, False), + # By not using `--no-preimports`, the entirety of `.submod1` is + # passed to `add_imported_function_or_module()` + (False, 'test_mod.submod1', True, False, + True, False, False, False), + (False, 'test_mod.submod2', False, True, + False, True, True, False), + (False, 'test_mod', False, True, + True, True, True, True), # Explicitly add all the modules via multiple `-p` flags, without # using the `--prof-imports` flag - (False, ['test_mod', 'test_mod.submod1,test_mod.submod2'], False, + (False, ['test_mod', 'test_mod.submod1,test_mod.submod2'], False, False, True, True, True, True), - (False, None, True, False, False, False, False), - (True, None, True, False, False, False, False)]) + (False, None, False, True, + False, False, False, False), + (True, None, False, True, + False, False, False, False)]) def test_autoprofile_exec_package( - use_kernprof_exec, prof_mod, prof_imports, + use_kernprof_exec, prof_mod, no_preimports, prof_imports, add_one, add_two, add_operator, main): """ Test the execution of a package. @@ -461,6 +470,8 @@ def test_autoprofile_exec_package( prof_mod = [prof_mod] for pm in prof_mod: args.extend(['-p', pm]) + if no_preimports: + args.append('--no-preimports') if prof_imports: args.append('--prof-imports') args.extend(['-l', '-m', 'test_mod', '1', '2', '3']) @@ -484,16 +495,28 @@ def test_autoprofile_exec_package( @pytest.mark.parametrize( - ['use_kernprof_exec', 'prof_mod', 'prof_imports', - 'add_one', 'add_two', 'add_four', 'add_operator', 'main'], - [(False, 'test_mod.submod2', False, False, True, False, False, False), - (False, 'test_mod.submod1', False, True, False, False, True, False), - (False, 'test_mod.subpkg.submod4', True, True, True, True, True, True), - (False, None, True, False, False, False, False, False), - (True, None, True, False, False, False, False, False)]) + ['use_kernprof_exec', 'prof_mod', 'no_preimports', 'prof_imports', + 'add_one', 'add_two', 'add_three', 'add_four', 'add_operator', 'main'], + [(False, 'test_mod.submod2,test_mod.subpkg.submod3.add_three', True, False, + False, True, False, False, False, False), + # By not using `--no-preimports`: + # - The entirety of `.submod2` is passed to + # `add_imported_function_or_module()` + # - Despite not having been imported anywhere, `add_three()` is + # still profiled + (False, 'test_mod.submod2,test_mod.subpkg.submod3.add_three', False, False, + False, True, True, False, True, False), + (False, 'test_mod.submod1', False, False, + True, False, False, False, True, False), + (False, 'test_mod.subpkg.submod4', False, True, + True, True, False, True, True, True), + (False, None, False, True, + False, False, False, False, False, False), + (True, None, False, True, + False, False, False, False, False, False)]) def test_autoprofile_exec_module( - use_kernprof_exec, prof_mod, prof_imports, - add_one, add_two, add_four, add_operator, main): + use_kernprof_exec, prof_mod, no_preimports, prof_imports, + add_one, add_two, add_three, add_four, add_operator, main): """ Test the execution of a module. """ @@ -506,6 +529,8 @@ def test_autoprofile_exec_module( args = [sys.executable, '-m', 'kernprof'] if prof_mod is not None: args.extend(['-p', prof_mod]) + if no_preimports: + args.append('--no-preimports') if prof_imports: args.append('--prof-imports') args.extend(['-l', '-m', 'test_mod.subpkg.submod4', '1', '2', '3']) @@ -524,6 +549,7 @@ def test_autoprofile_exec_module( assert ('Function: add_one' in raw_output) == add_one assert ('Function: add_two' in raw_output) == add_two + assert ('Function: add_three' in raw_output) == add_three assert ('Function: add_four' in raw_output) == add_four assert ('Function: add_operator' in raw_output) == add_operator assert ('Function: _main' in raw_output) == main @@ -625,78 +651,14 @@ def test_autoprofile_from_inlined_script(outfile, expected_outfile) -> None: @pytest.mark.parametrize( - ['prof_mod', 'eager_preimports', - 'add_one', 'add_two', 'add_three', 'add_four', 'add_operator', 'main'], - # Test that `--eager-preimports` know to exclude the script run - # (so as not to inadvertantly run it twice) - [('script.py', None, False, False, False, False, False, True), - (None, 'script.py', False, False, False, False, False, True), - # Test explicitly passing targets to `--eager-preimports` - (['test_mod.submod1,test_mod.submod2', 'test_mod.subpkg.submod4'], None, - True, True, False, False, True, False), - (['test_mod.submod1,test_mod.submod2'], ['test_mod.subpkg.submod4'], - True, True, False, True, True, False), - (None, ['test_mod.submod1,test_mod.submod2', 'test_mod.subpkg.submod4'], - True, True, False, True, True, False), - # Test implicitly passing targets to `--eager-preimports` - (['test_mod.submod1,test_mod.submod2', 'test_mod.subpkg.submod4'], True, - True, True, False, True, True, False)]) -def test_autoprofile_eager_preimports( - prof_mod, eager_preimports, - add_one, add_two, add_three, add_four, add_operator, main): - """ - Test eager imports with the `-e`/`--eager-preimports` flag. - """ - with tempfile.TemporaryDirectory() as tmpdir: - temp_dpath = ub.Path(tmpdir) - _write_demo_module(temp_dpath) - - args = [sys.executable, '-m', 'kernprof'] - if prof_mod is not None: - if isinstance(prof_mod, str): - prof_mod = [prof_mod] - for target in prof_mod: - args.extend(['-p', target]) - if eager_preimports in (True,): - args.append('-e') - elif eager_preimports is not None: - if isinstance(eager_preimports, str): - eager_preimports = [eager_preimports] - for target in eager_preimports: - args.extend(['-e', target]) - args.extend(['-l', 'script.py']) - proc = ub.cmd(args, cwd=temp_dpath, verbose=2) - # Check that pre-imports don't accidentally run the code twice - assert proc.stdout.count('7.9') == 1 - print(proc.stdout) - print(proc.stderr) - proc.check_returncode() - - prof = temp_dpath / 'script.py.lprof' - - args = [sys.executable, '-m', 'line_profiler', os.fspath(prof)] - proc = ub.cmd(args, cwd=temp_dpath) - raw_output = proc.stdout - print(raw_output) - proc.check_returncode() - - assert ('Function: add_one' in raw_output) == add_one - assert ('Function: add_two' in raw_output) == add_two - assert ('Function: add_three' in raw_output) == add_three - assert ('Function: add_four' in raw_output) == add_four - assert ('Function: add_operator' in raw_output) == add_operator - assert ('Function: main' in raw_output) == main - - -@pytest.mark.parametrize( - ('eager_preimports, function, method, class_method, static_method, ' + ('prof_mod, function, method, class_method, static_method, ' 'descriptor'), [('my_module', True, True, True, True, True), # `function()` included in profiling via `Class.partial_method()` ('my_module.Class', True, True, True, True, True), ('my_module.Class.descriptor', False, False, False, False, True)]) def test_autoprofile_callable_wrapper_objects( - eager_preimports, function, method, class_method, static_method, + prof_mod, function, method, class_method, static_method, descriptor): """ Test that on-import profiling catches various callable-wrapper @@ -747,7 +709,7 @@ def descriptor(self): with ub.ChDir(temp_dpath): args = [sys.executable, '-m', 'kernprof', - '-e', eager_preimports, '-lv', 'script.py'] + '-p', prof_mod, '-lv', 'script.py'] python_path = os.environ.get('PYTHONPATH') if python_path: python_path = '{}:{}'.format(path, python_path) From 1620fee1c3fe15a8aee9cd45dc5b408a5094d5ff Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 18 May 2025 08:35:56 +0200 Subject: [PATCH 18/70] CI fix --- kernprof.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kernprof.py b/kernprof.py index f407f537..46a5f636 100755 --- a/kernprof.py +++ b/kernprof.py @@ -738,6 +738,9 @@ def _main(options, module=False): print(f'{py_exe} -m pstats "{options.outfile}"') else: print(f'{py_exe} -m line_profiler -rmt "{options.outfile}"') + # Fully disable the profiler + for _ in range(prof.enable_count): + prof.disable_by_count() # Restore the state of the global `@line_profiler.profile` if global_profiler: install_profiler(None) From 831e52bfa4b46a7464b1c92ae88a25b3b3de1ebf Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 18 May 2025 23:09:54 +0200 Subject: [PATCH 19/70] `line_profiler` refactoring line_profiler/line_profiler.py[i]::LineProfiler _add_namespace() - Signature changes: - Renamed: `match_scope` -> `scoping_policy` - Renamed: `duplicate_tracker` -> `seen` - `seen` now a keyword-only parameter - Internal refactoring to call the methods directly instead of pre-fetching them _add_{class,module}_filter(), add_{class,module}() Renamed parameter `match_scope` -> `scoping_policy` line_profiler/autoprofile/line_profiler_utils.py[i] add_imported_function_or_module() Renamed parameter: `match_scope` -> `scoping_policy` tests/test_explicit_profile.py::test_profiler_scope_matching() Updated because of the change in the signature of `LineProfiler.add_class()` and `.add_module()` --- .../autoprofile/line_profiler_utils.py | 10 ++-- .../autoprofile/line_profiler_utils.pyi | 8 ++- line_profiler/line_profiler.py | 57 ++++++++++--------- line_profiler/line_profiler.pyi | 6 +- tests/test_explicit_profile.py | 8 +-- 5 files changed, 46 insertions(+), 43 deletions(-) diff --git a/line_profiler/autoprofile/line_profiler_utils.py b/line_profiler/autoprofile/line_profiler_utils.py index 5867587a..f1e4714a 100644 --- a/line_profiler/autoprofile/line_profiler_utils.py +++ b/line_profiler/autoprofile/line_profiler_utils.py @@ -2,7 +2,7 @@ def add_imported_function_or_module(self, item, *, - match_scope='siblings', wrap=False): + scoping_policy='siblings', wrap=False): """ Method to add an object to `LineProfiler` to be profiled. @@ -13,8 +13,8 @@ def add_imported_function_or_module(self, item, *, Args: item (Union[Callable, Type, ModuleType]): Object to be profiled. - match_scope (Literal['exact', 'siblings', 'descendants', - 'none']): + scoping_policy (Literal['exact', 'siblings', 'descendants', + 'none']): Whether (and how) to match the scope of member classes to `item` (if a class or module) and decide on whether to add them: @@ -39,9 +39,9 @@ def add_imported_function_or_module(self, item, *, `LineProfiler.add_callable()`, `.add_module()`, `.add_class()` """ if inspect.isclass(item): - count = self.add_class(item, match_scope=match_scope, wrap=wrap) + count = self.add_class(item, scoping_policy=scoping_policy, wrap=wrap) elif inspect.ismodule(item): - count = self.add_module(item, match_scope=match_scope, wrap=wrap) + count = self.add_module(item, scoping_policy=scoping_policy, wrap=wrap) else: try: count = self.add_callable(item) diff --git a/line_profiler/autoprofile/line_profiler_utils.pyi b/line_profiler/autoprofile/line_profiler_utils.pyi index cfbef479..bb554840 100644 --- a/line_profiler/autoprofile/line_profiler_utils.pyi +++ b/line_profiler/autoprofile/line_profiler_utils.pyi @@ -2,13 +2,15 @@ from types import ModuleType from typing import overload, Any, Literal, TYPE_CHECKING if TYPE_CHECKING: # Stub-only annotations - from ..line_profiler import CLevelCallable, CallableLike, MatchScopeOption + from ..line_profiler import ( + CLevelCallable, CallableLike, ScopingPolicyOption, + ) @overload def add_imported_function_or_module( self, item: CLevelCallable | Any, - match_scope: MatchScopeOption = 'siblings', + scoping_policy: ScopingPolicyOption = 'siblings', wrap: bool = False) -> Literal[0]: ... @@ -16,6 +18,6 @@ def add_imported_function_or_module( @overload def add_imported_function_or_module( self, item: CallableLike | type | ModuleType, - match_scope: MatchScopeOption = 'siblings', + scoping_policy: ScopingPolicyOption = 'siblings', wrap: bool = False) -> Literal[0, 1]: ... diff --git a/line_profiler/line_profiler.py b/line_profiler/line_profiler.py index cba36364..6e5e84b2 100755 --- a/line_profiler/line_profiler.py +++ b/line_profiler/line_profiler.py @@ -181,38 +181,39 @@ def print_stats(self, stream=None, output_unit=None, stripzeros=False, stream=stream, stripzeros=stripzeros, details=details, summarize=summarize, sort=sort, rich=rich) - def _add_namespace(self, duplicate_tracker, namespace, *, - match_scope='none', wrap=False): + def _add_namespace(self, namespace, *, + seen=None, scoping_policy='none', wrap=False): + if seen is None: + seen = set() count = 0 - add_cls = functools.partial(self._add_namespace, duplicate_tracker, - match_scope=match_scope, wrap=wrap) - add_func = self.add_callable - remember = duplicate_tracker.add - wrap_func = self.wrap_callable + add_cls = functools.partial(self._add_namespace, + seen=seen, + scoping_policy=scoping_policy, + wrap=wrap) wrap_failures = {} - if match_scope == 'none': + if scoping_policy == 'none': def check(*_): return True elif isinstance(namespace, type): - check = self._add_class_filter(namespace, match_scope) + check = self._add_class_filter(namespace, scoping_policy) else: - check = self._add_module_filter(namespace, match_scope) + check = self._add_module_filter(namespace, scoping_policy) for attr, value in vars(namespace).items(): - if id(value) in duplicate_tracker: + if id(value) in seen: continue - remember(id(value)) + seen.add(id(value)) if isinstance(value, type): if check(value) and add_cls(value): count += 1 continue try: - if not add_func(value): + if not self.add_callable(value): continue except TypeError: # Not a callable (wrapper) continue if wrap: - wrapper = wrap_func(value) + wrapper = self.wrap_callable(value) if wrapper is not value: try: setattr(namespace, attr, wrapper) @@ -232,7 +233,7 @@ def check(*_): return count @staticmethod - def _add_module_filter(mod, match_scope): + def _add_module_filter(mod, scoping_policy): def match_prefix(s, prefix, sep='.'): return s == prefix or s.startswith(prefix + sep) @@ -252,10 +253,10 @@ def class_is_cousin(other): 'descendants': class_is_descendant, 'siblings': (class_is_cousin # Only if a pkg if basename else - class_is_descendant)}[match_scope] + class_is_descendant)}[scoping_policy] @staticmethod - def _add_class_filter(cls, match_scope): + def _add_class_filter(cls, scoping_policy): def class_is_child(other): if not modules_are_equal(other): return False @@ -271,9 +272,9 @@ def class_is_descendant(other): return {'exact': class_is_child, 'descendants': class_is_descendant, - 'siblings': modules_are_equal}[match_scope] + 'siblings': modules_are_equal}[scoping_policy] - def add_class(self, cls, *, match_scope='siblings', wrap=False): + def add_class(self, cls, *, scoping_policy='siblings', wrap=False): """ Add the members (callables (wrappers), methods, classes, ...) in a class' local namespace and profile them. @@ -281,8 +282,8 @@ def add_class(self, cls, *, match_scope='siblings', wrap=False): Args: cls (type): Class to be profiled. - match_scope (Literal['exact', 'siblings', 'descendants', - 'none']): + scoping_policy (Literal['exact', 'siblings', 'descendants', + 'none']): Whether (and how) to match the scope of member classes and decide on whether to add them: - 'exact': only add classes defined locally in this @@ -303,10 +304,10 @@ def add_class(self, cls, *, match_scope='siblings', wrap=False): n (int): Number of members added to the profiler. """ - return self._add_namespace(set(), cls, - match_scope=match_scope, wrap=wrap) + return self._add_namespace(cls, + scoping_policy=scoping_policy, wrap=wrap) - def add_module(self, mod, *, match_scope='siblings', wrap=False): + def add_module(self, mod, *, scoping_policy='siblings', wrap=False): """ Add the members (callables (wrappers), methods, classes, ...) in a module's local namespace and profile them. @@ -314,8 +315,8 @@ def add_module(self, mod, *, match_scope='siblings', wrap=False): Args: mod (ModuleType): Module to be profiled. - match_scope (Literal['exact', 'siblings', 'descendants', - 'none']): + scoping_policy (Literal['exact', 'siblings', 'descendants', + 'none']): Whether (and how) to match the scope of member classes and decide on whether to add them: - 'exact': only add classes defined locally in this @@ -336,8 +337,8 @@ def add_module(self, mod, *, match_scope='siblings', wrap=False): n (int): Number of members added to the profiler. """ - return self._add_namespace(set(), mod, - match_scope=match_scope, wrap=wrap) + return self._add_namespace(mod, + scoping_policy=scoping_policy, wrap=wrap) def _get_wrapper_info(self, func): info = getattr(func, self._profiler_wrapped_marker, None) diff --git a/line_profiler/line_profiler.pyi b/line_profiler/line_profiler.pyi index 7d4c70b1..7c52f231 100644 --- a/line_profiler/line_profiler.pyi +++ b/line_profiler/line_profiler.pyi @@ -19,7 +19,7 @@ CLevelCallable = TypeVar('CLevelCallable', CallableLike = TypeVar('CallableLike', FunctionType, partial, property, cached_property, MethodType, staticmethod, classmethod, partialmethod) -MatchScopeOption = Literal['exact', 'descendants', 'siblings', 'none'] +ScopingPolicyOption = Literal['exact', 'descendants', 'siblings', 'none'] def is_c_level_callable(func: Any) -> TypeGuard[CLevelCallable]: @@ -64,12 +64,12 @@ class LineProfiler(CLineProfiler, ByCountProfilerMixin): ... def add_module(self, mod: ModuleType, *, - match_scope: MatchScopeOption = 'siblings', + scoping_policy: ScopingPolicyOption = 'siblings', wrap: bool = False) -> int: ... def add_class(self, cls: type, *, - match_scope: MatchScopeOption = 'siblings', + scoping_policy: ScopingPolicyOption = 'siblings', wrap: bool = False) -> int: ... diff --git a/tests/test_explicit_profile.py b/tests/test_explicit_profile.py index 640d7549..8cb47506 100644 --- a/tests/test_explicit_profile.py +++ b/tests/test_explicit_profile.py @@ -584,7 +584,7 @@ def method(self): @pytest.mark.parametrize( - ('match_scope', 'add_module_targets', 'add_class_targets'), + ('scoping_policy', 'add_module_targets', 'add_class_targets'), [('exact', {'class2_method', 'child_class2_method'}, {'class3_method', 'child_class3_method'}), @@ -604,7 +604,7 @@ def method(self): {'child_class1_method', 'class3_method', 'child_class3_method', 'other_class3_method'})]) def test_profiler_scope_matching(monkeypatch, - match_scope, + scoping_policy, add_module_targets, add_class_targets): """ @@ -677,13 +677,13 @@ def other_class3_method(self): # Add a module profile = LineProfiler() - profile.add_module(subpkg2, match_scope=match_scope) + profile.add_module(subpkg2, scoping_policy=scoping_policy) assert len(profile.functions) == len(add_module_targets) added = {func.__name__ for func in profile.functions} assert added == set(add_module_targets) # Add a class profile = LineProfiler() - profile.add_class(subpkg2.Class3, match_scope=match_scope) + profile.add_class(subpkg2.Class3, scoping_policy=scoping_policy) assert len(profile.functions) == len(add_class_targets) added = {func.__name__ for func in profile.functions} assert added == set(add_class_targets) From f6cf3052a6e76c7dac720b2d33ed6e827274afbf Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 19 May 2025 00:36:14 +0200 Subject: [PATCH 20/70] `line_profiler.line_profiler_utils` module line_profiler/line_profiler_utils.py[i] New module for miscellaneous utilities used by `line_profiler`; currently containing `StringEnum`, a backport-slash-extension of `enum.StrEnum` in Python 3.11+ --- line_profiler/line_profiler_utils.py | 73 +++++++++++++++++++++++++++ line_profiler/line_profiler_utils.pyi | 27 ++++++++++ 2 files changed, 100 insertions(+) create mode 100644 line_profiler/line_profiler_utils.py create mode 100644 line_profiler/line_profiler_utils.pyi diff --git a/line_profiler/line_profiler_utils.py b/line_profiler/line_profiler_utils.py new file mode 100644 index 00000000..0d903888 --- /dev/null +++ b/line_profiler/line_profiler_utils.py @@ -0,0 +1,73 @@ +""" +Miscellaneous utilities that :py:mod:`line_profiler` uses. +""" +import enum + + +class _StrEnumBase(str, enum.Enum): + """ + Base class mimicking :py:class:`enum.StrEnum` in Python 3.11+. + + Example + ------- + >>> import enum + >>> + >>> + >>> class MyEnum(_StrEnumBase): + ... foo = enum.auto() + ... BAR = enum.auto() + ... + >>> + >>> MyEnum.foo + + >>> MyEnum('bar') + + >>> MyEnum('baz') + Traceback (most recent call last): + ... + ValueError: 'baz' is not a valid MyEnum + """ + @staticmethod + def _generate_next_value_(name, *_, **__): + return name.lower() + + def __eq__(self, other): + return self.value == other + + def __str__(self): + return self.value + + +class StringEnum(getattr(enum, 'StrEnum', _StrEnumBase)): + """ + Convenience wrapper around :py:class:`enum.StrEnum`. + + Example + ------- + >>> import enum + >>> + >>> + >>> class MyEnum(StringEnum): + ... foo = enum.auto() + ... BAR = enum.auto() + ... + >>> + >>> MyEnum.foo + + >>> MyEnum('bar') + + >>> bar = MyEnum('BAR') # Case-insensitive + >>> bar + + >>> assert isinstance(bar, str) + >>> assert bar == 'bar' + >>> str(bar) + 'bar' + """ + @classmethod + def _missing_(cls, value): + if not isinstance(value, str): + return None + members = {name.casefold(): instance + for name, instance in cls.__members__.items()} + return members.get(value.casefold()) diff --git a/line_profiler/line_profiler_utils.pyi b/line_profiler/line_profiler_utils.pyi new file mode 100644 index 00000000..ad7447ff --- /dev/null +++ b/line_profiler/line_profiler_utils.pyi @@ -0,0 +1,27 @@ +import enum +from typing import Type, TypeVar, Union +try: + from typing import Self # type: ignore[attr-defined] # noqa: F401 +except ImportError: # Python < 3.11 + from typing_extensions import Self # noqa: F401 + + +# Note: `mypy` tries to read this class as a free-standing enum +# (instead of an `enum.Enum` subclass that string enums are to inherit +# from), and complains that it has no members -- so silence that + + +class StringEnum(str, enum.Enum): # type: ignore[misc] + @staticmethod + def _generate_next_value_(name: str, *_, **__) -> str: + ... + + def __eq__(self, other) -> bool: + ... + + def __str__(self) -> str: + ... + + @classmethod + def _missing_(cls, value) -> Union[Self, None]: + ... From b59dffcfab85d94777b7f1142158b113446c2276 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 19 May 2025 02:51:34 +0200 Subject: [PATCH 21/70] More refactoring line_profiler/autoprofile/line_profiler_utils.py[i] add_imported_function_or_module() - Updated type annotation and default value for `scoping_policy` - Reformatted docstring to be more `sphinx`-friendly line_profiler/line_profiler.py[i] C_LEVEL_CALLABLE_TYPES Renamed from `c_level_callable_types` is_c_level_callable() Updated return annotation to use `TypeIs` instead of `TypeGuard` since neither is in 3.8 anyway ScopingPolicy New string enum documenting and delineating valid values for `LineProfiler.add_*(scoping_policy=...)` LineProfiler.add_class(), .add_module() - Updated docstrings and type annotations - Now converting arugment `scoping_policy` to `ScopingPolicy` objects LineProfiler._add_class_filter(), ._add_module_filter() Moved implementations to `ScopingPolicy` LineProfiler.load_stats(), .dump_stats() Reformatted docstrings to be more `sphinx`-friendly --- .../autoprofile/line_profiler_utils.py | 41 ++- .../autoprofile/line_profiler_utils.pyi | 6 +- line_profiler/line_profiler.py | 240 ++++++++++++------ line_profiler/line_profiler.pyi | 37 ++- 4 files changed, 208 insertions(+), 116 deletions(-) diff --git a/line_profiler/autoprofile/line_profiler_utils.py b/line_profiler/autoprofile/line_profiler_utils.py index f1e4714a..355af7ee 100644 --- a/line_profiler/autoprofile/line_profiler_utils.py +++ b/line_profiler/autoprofile/line_profiler_utils.py @@ -1,33 +1,30 @@ import inspect +from ..line_profiler import ScopingPolicy -def add_imported_function_or_module(self, item, *, - scoping_policy='siblings', wrap=False): +def add_imported_function_or_module( + self, item, *, + scoping_policy=ScopingPolicy.SIBLINGS, wrap=False): """ - Method to add an object to `LineProfiler` to be profiled. + Method to add an object to + :py:class:`~.line_profiler.LineProfiler` to be profiled. - This method is used to extend an instance of `LineProfiler` so it - can identify whether an object is a callable (wrapper), a class, or - a module, and handle its profiling accordingly. + This method is used to extend an instance of + :py:class:`~.line_profiler.LineProfiler` so it can identify whether + an object is a callable (wrapper), a class, or a module, and handle + its profiling accordingly. Args: item (Union[Callable, Type, ModuleType]): Object to be profiled. - scoping_policy (Literal['exact', 'siblings', 'descendants', - 'none']): + scoping_policy (Union[ScopingPolicy, str]): Whether (and how) to match the scope of member classes to - `item` (if a class or module) and decide on whether to add - them: - - 'exact': only add classes defined locally in the body of - `item` - - 'descendants': only add locally-defined classes and - classes defined in submodules or locally-defined class - bodies, and so on. - - 'siblings': only add classes fulfilling 'descendants', - or defined in the same module as `item` (if a class) or in - sibling modules and subpackages to `item` (if a module) - - 'none': don't check scopes and add all classes in the - namespace + ``item`` (if a class or module) and decide on whether to add + them; + see the documentation for :py:class:`~.ScopingPolicy` for + details. + Strings are converted to :py:class:`~.ScopingPolicy` + instances in a case-insensitive manner. wrap (bool): Whether to replace the wrapped members with wrappers which automatically enable/disable the profiler when called. @@ -36,7 +33,9 @@ def add_imported_function_or_module(self, item, *, 1 if any function is added to the profiler, 0 otherwise. See also: - `LineProfiler.add_callable()`, `.add_module()`, `.add_class()` + :py:meth:`.LineProfiler.add_callable()`, + :py:meth:`.LineProfiler.add_module()`, + :py:meth:`.LineProfiler.add_class()` """ if inspect.isclass(item): count = self.add_class(item, scoping_policy=scoping_policy, wrap=wrap) diff --git a/line_profiler/autoprofile/line_profiler_utils.pyi b/line_profiler/autoprofile/line_profiler_utils.pyi index bb554840..0d859824 100644 --- a/line_profiler/autoprofile/line_profiler_utils.pyi +++ b/line_profiler/autoprofile/line_profiler_utils.pyi @@ -3,14 +3,14 @@ from typing import overload, Any, Literal, TYPE_CHECKING if TYPE_CHECKING: # Stub-only annotations from ..line_profiler import ( - CLevelCallable, CallableLike, ScopingPolicyOption, + CLevelCallable, CallableLike, ScopingPolicy, ) @overload def add_imported_function_or_module( self, item: CLevelCallable | Any, - scoping_policy: ScopingPolicyOption = 'siblings', + scoping_policy: ScopingPolicy | str = ScopingPolicy.SIBLINGS, wrap: bool = False) -> Literal[0]: ... @@ -18,6 +18,6 @@ def add_imported_function_or_module( @overload def add_imported_function_or_module( self, item: CallableLike | type | ModuleType, - scoping_policy: ScopingPolicyOption = 'siblings', + scoping_policy: ScopingPolicy | str = ScopingPolicy.SIBLINGS, wrap: bool = False) -> Literal[0, 1]: ... diff --git a/line_profiler/line_profiler.py b/line_profiler/line_profiler.py index 6e5e84b2..e44ce565 100755 --- a/line_profiler/line_profiler.py +++ b/line_profiler/line_profiler.py @@ -14,6 +14,7 @@ import types import warnings from argparse import ArgumentError, ArgumentParser +from enum import auto try: from ._line_profiler import LineProfiler as CLineProfiler @@ -22,6 +23,7 @@ 'The line_profiler._line_profiler c-extension is not importable. ' f'Has it been compiled? Underlying error is ex={ex!r}' ) +from .line_profiler_utils import StringEnum from .profiler_mixin import (ByCountProfilerMixin, is_property, is_cached_property, is_boundmethod, is_classmethod, is_staticmethod, @@ -33,7 +35,7 @@ # These objects are callables, but are defined in C so we can't handle # them anyway -c_level_callable_types = (types.BuiltinFunctionType, +C_LEVEL_CALLABLE_TYPES = (types.BuiltinFunctionType, types.BuiltinMethodType, types.ClassMethodDescriptorType, types.MethodDescriptorType, @@ -50,7 +52,7 @@ def is_c_level_callable(func): Whether a callable is defined at the C level (and is thus non-profilable). """ - return isinstance(func, c_level_callable_types) + return isinstance(func, C_LEVEL_CALLABLE_TYPES) def load_ipython_extension(ip): @@ -105,6 +107,136 @@ def __init__(self, func, profiler_id): self.profiler_id = profiler_id +class ScopingPolicy(StringEnum): + """ + :py:class:`StrEnum` for scoping policies, that is, how nested + namespaces (classes and modules) are descended into when using + :py:meth:`LineProfiler.add_class`, + :py:meth:`LineProfiler.add_module`, and + :py:func:`~.add_imported_function_or_module()`. + + Available policies are: + + :py:attr:`ScopingPolicy.EXACT` + Only add classes defined locally in the very module, or + the very class as its "inner classes" + :py:attr:`ScopingPolicy.DESCENDANTS` + Only add locally-defined classes (see :py:attr:`EXACT`), + their locally-defined classes, and so on + :py:attr:`ScopingPolicy.SIBLINGS` + Only add classes fulfilling :py:attr:`DESCENDANTS`, or are + defined in the same module as this very class, or are + defined in sibling modules and subpackages (if a part of a + package) to this very module + :py:attr:`ScopingPolicy.NONE` + Don't check scopes and add all classes in the local + namespace of the class/module + """ + EXACT = auto() + DESCENDANTS = auto() + SIBLINGS = auto() + NONE = auto() + + def __init_subclass__(cls, *args, **kwargs): + """ + Call :py:meth:`_check_class`. + """ + super().__init_subclass__(*args, **kwargs) + cls._check_class() + + @classmethod + def _check_class(cls): + """ + Verify that :py:meth:`_add_module_filter` and + :py:meth:`_add_class_filter` returns a callable for all policy + values. + """ + mock_module = types.ModuleType('mock_module') + + class MockClass: + pass + + for member in cls.__members__.values(): + assert callable(member._add_module_filter(mock_module)) + assert callable(member._add_class_filter(MockClass)) + + @staticmethod + def _no_op(_): + """ + Filter that is always true. + """ + return True + + def _add_module_filter(self, mod): + """ + Args: + mod (ModuleType): + Module to be profiled. + + Returns: + func (Callable[[type], bool]): + Filter callable returning whether the argument, a class + in the local namespace of ``mod``, should be descended + into and added via :py:meth:`LineProfiler.add_class` + """ + def match_prefix(s, prefix, sep='.'): + return s == prefix or s.startswith(prefix + sep) + + def class_is_child(other): + return other.__module__ == mod.__name__ + + def class_is_descendant(other): + return match_prefix(other.__module__, mod.__name__) + + def class_is_cousin(other): + if class_is_descendant(other): + return True + return match_prefix(other.__module__, parent) + + parent, _, basename = mod.__name__.rpartition('.') + return {'exact': class_is_child, + 'descendants': class_is_descendant, + 'siblings': (class_is_cousin # Only if a pkg + if basename else + class_is_descendant), + 'none': self._no_op}[self.value] + + def _add_class_filter(self, cls): + """ + Args: + cls (type): + Class to be profiled. + + Returns: + func (Callable[[type], bool]): + Filter callable returning whether the argument, a class + in the local namespace of ``cls``, should be descended + into and added via :py:meth:`LineProfiler.add_class` + """ + def class_is_child(other): + if not modules_are_equal(other): + return False + return other.__qualname__ == f'{cls.__qualname__}.{other.__name__}' + + def modules_are_equal(other): # = sibling check + return cls.__module__ == other.__module__ + + def class_is_descendant(other): + if not modules_are_equal(other): + return False + return other.__qualname__.startswith(cls.__qualname__ + '.') + + return {'exact': class_is_child, + 'descendants': class_is_descendant, + 'siblings': modules_are_equal, + 'none': self._no_op}[self.value] + + +# Sanity check in case we extended `ScopingPolicy` and forgot to update +# the corresponding methods +ScopingPolicy._check_class() + + class LineProfiler(CLineProfiler, ByCountProfilerMixin): """ A profiler that records the execution times of individual lines. @@ -165,8 +297,8 @@ def add_callable(self, func): return 1 if nadded else 0 def dump_stats(self, filename): - """ Dump a representation of the data to a file as a pickled LineStats - object from `get_stats()`. + """ Dump a representation of the data to a file as a pickled + :py:class:`~.LineStats` object from :py:meth:`~.get_stats()`. """ lstats = self.get_stats() with open(filename, 'wb') as f: @@ -181,8 +313,9 @@ def print_stats(self, stream=None, output_unit=None, stripzeros=False, stream=stream, stripzeros=stripzeros, details=details, summarize=summarize, sort=sort, rich=rich) - def _add_namespace(self, namespace, *, - seen=None, scoping_policy='none', wrap=False): + def _add_namespace( + self, namespace, *, + seen=None, scoping_policy=ScopingPolicy.NONE, wrap=False): if seen is None: seen = set() count = 0 @@ -191,13 +324,10 @@ def _add_namespace(self, namespace, *, scoping_policy=scoping_policy, wrap=wrap) wrap_failures = {} - if scoping_policy == 'none': - def check(*_): - return True - elif isinstance(namespace, type): - check = self._add_class_filter(namespace, scoping_policy) + if isinstance(namespace, type): + check = scoping_policy._add_class_filter(namespace) else: - check = self._add_module_filter(namespace, scoping_policy) + check = scoping_policy._add_module_filter(namespace) for attr, value in vars(namespace).items(): if id(value) in seen: @@ -232,49 +362,8 @@ def check(*_): warnings.warn(msg, stacklevel=2) return count - @staticmethod - def _add_module_filter(mod, scoping_policy): - def match_prefix(s, prefix, sep='.'): - return s == prefix or s.startswith(prefix + sep) - - def class_is_child(other): - return other.__module__ == mod.__name__ - - def class_is_descendant(other): - return match_prefix(other.__module__, mod.__name__) - - def class_is_cousin(other): - if class_is_descendant(other): - return True - return match_prefix(other.__module__, parent) - - parent, _, basename = mod.__name__.rpartition('.') - return {'exact': class_is_child, - 'descendants': class_is_descendant, - 'siblings': (class_is_cousin # Only if a pkg - if basename else - class_is_descendant)}[scoping_policy] - - @staticmethod - def _add_class_filter(cls, scoping_policy): - def class_is_child(other): - if not modules_are_equal(other): - return False - return other.__qualname__ == f'{cls.__qualname__}.{other.__name__}' - - def modules_are_equal(other): # = sibling check - return cls.__module__ == other.__module__ - - def class_is_descendant(other): - if not modules_are_equal(other): - return False - return other.__qualname__.startswith(cls.__qualname__ + '.') - - return {'exact': class_is_child, - 'descendants': class_is_descendant, - 'siblings': modules_are_equal}[scoping_policy] - - def add_class(self, cls, *, scoping_policy='siblings', wrap=False): + def add_class( + self, cls, *, scoping_policy=ScopingPolicy.SIBLINGS, wrap=False): """ Add the members (callables (wrappers), methods, classes, ...) in a class' local namespace and profile them. @@ -282,19 +371,12 @@ def add_class(self, cls, *, scoping_policy='siblings', wrap=False): Args: cls (type): Class to be profiled. - scoping_policy (Literal['exact', 'siblings', 'descendants', - 'none']): + scoping_policy (Union[str, ScopingPolicy]): Whether (and how) to match the scope of member classes and decide on whether to add them: - - 'exact': only add classes defined locally in this - namespace, i.e. in the body of `cls`, as "inner - classes" - - 'descendants': only add "inner classes", their "inner - classes", and so on. - - 'siblings': only add classes fulfilling 'descendants', - or defined in the same module as `cls` - - 'none': don't check scopes and add all classes in the - namespace + see the documentation for :py:class:`ScopingPolicy`. + Strings are converted to :py:class:`ScopingPolicy` + instances in a case-insensitive manner. wrap (bool): Whether to replace the wrapped members with wrappers which automatically enable/disable the profiler when @@ -304,10 +386,12 @@ def add_class(self, cls, *, scoping_policy='siblings', wrap=False): n (int): Number of members added to the profiler. """ + scoping_policy = ScopingPolicy(scoping_policy) return self._add_namespace(cls, scoping_policy=scoping_policy, wrap=wrap) - def add_module(self, mod, *, scoping_policy='siblings', wrap=False): + def add_module( + self, mod, *, scoping_policy=ScopingPolicy.SIBLINGS, wrap=False): """ Add the members (callables (wrappers), methods, classes, ...) in a module's local namespace and profile them. @@ -315,19 +399,12 @@ def add_module(self, mod, *, scoping_policy='siblings', wrap=False): Args: mod (ModuleType): Module to be profiled. - scoping_policy (Literal['exact', 'siblings', 'descendants', - 'none']): + scoping_policy (Union[str, ScopingPolicy]): Whether (and how) to match the scope of member classes - and decide on whether to add them: - - 'exact': only add classes defined locally in this - namespace, i.e. in the body of `mod` - - 'descendants': only add locally-defined classes, - classes locally defined in their bodies, and so on - - 'siblings': only add classes fulfilling 'descendants', - or defined in sibling modules/subpackages to `mod` (if - `mod` is part of a package) - - 'none': don't check scopes and add all classes in the - namespace + and decide on whether to add them; + see the documentation for :py:class:`ScopingPolicy`. + Strings are converted to :py:class:`ScopingPolicy` + instances in a case-insensitive manner. wrap (bool): Whether to replace the wrapped members with wrappers which automatically enable/disable the profiler when @@ -337,6 +414,7 @@ def add_module(self, mod, *, scoping_policy='siblings', wrap=False): n (int): Number of members added to the profiler. """ + scoping_policy = ScopingPolicy(scoping_policy) return self._add_namespace(mod, scoping_policy=scoping_policy, wrap=wrap) @@ -660,8 +738,8 @@ def show_text(stats, unit, output_unit=None, stream=None, stripzeros=False, def load_stats(filename): - """ Utility function to load a pickled LineStats object from a given - filename. + """ Utility function to load a pickled :py:class:`~.LineStats` + object from a given filename. """ with open(filename, 'rb') as f: return pickle.load(f) diff --git a/line_profiler/line_profiler.pyi b/line_profiler/line_profiler.pyi index 7c52f231..38865d95 100644 --- a/line_profiler/line_profiler.pyi +++ b/line_profiler/line_profiler.pyi @@ -1,15 +1,21 @@ +import io +from enum import auto from functools import cached_property, partial, partialmethod from inspect import isfunction as is_function from types import (FunctionType, MethodType, ModuleType, BuiltinFunctionType, BuiltinMethodType, ClassMethodDescriptorType, MethodDescriptorType, MethodWrapperType, WrapperDescriptorType) -from typing import (overload, - Any, Literal, Callable, List, Tuple, TypeGuard, TypeVar) -import io +from typing import overload, Any, Literal, Callable, List, Tuple, TypeVar +try: + from typing import ( # type: ignore[attr-defined] # noqa: F401 + TypeIs) +except ImportError: # Python < 3.13 + from typing_extensions import TypeIs # noqa: F401 +from _typeshed import Incomplete from ._line_profiler import LineProfiler as CLineProfiler +from .line_profiler_utils import StringEnum from .profiler_mixin import ByCountProfilerMixin -from _typeshed import Incomplete CLevelCallable = TypeVar('CLevelCallable', @@ -22,7 +28,7 @@ CallableLike = TypeVar('CallableLike', ScopingPolicyOption = Literal['exact', 'descendants', 'siblings', 'none'] -def is_c_level_callable(func: Any) -> TypeGuard[CLevelCallable]: +def is_c_level_callable(func: Any) -> TypeIs[CLevelCallable]: ... @@ -30,6 +36,13 @@ def load_ipython_extension(ip) -> None: ... +class ScopingPolicy(StringEnum): + EXACT = auto() + DESCENDANTS = auto() + SIBLINGS = auto() + NONE = auto() + + class LineProfiler(CLineProfiler, ByCountProfilerMixin): @overload def __call__(self, # type: ignore[overload-overlap] @@ -63,14 +76,16 @@ class LineProfiler(CLineProfiler, ByCountProfilerMixin): rich: bool = ...) -> None: ... - def add_module(self, mod: ModuleType, *, - scoping_policy: ScopingPolicyOption = 'siblings', - wrap: bool = False) -> int: + def add_module( + self, mod: ModuleType, *, + scoping_policy: ScopingPolicy | str = ScopingPolicy.SIBLINGS, + wrap: bool = False) -> int: ... - def add_class(self, cls: type, *, - scoping_policy: ScopingPolicyOption = 'siblings', - wrap: bool = False) -> int: + def add_class( + self, cls: type, *, + scoping_policy: ScopingPolicy | str = ScopingPolicy.SIBLINGS, + wrap: bool = False) -> int: ... From 86418728d2721bbe7ea33435c942077710d91da6 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 19 May 2025 03:09:24 +0200 Subject: [PATCH 22/70] Streamlined `tests/test_eager_preimports.py` tests/test_eager_preimports.py create_doctest_wrapper(), regularize_doctests() Removed since the `line_profiler.autoprofile.eager_preimports` doctests are already run in CI, despite not being covered in `run_tests.py` by default --- tests/test_eager_preimports.py | 178 ++------------------------------- 1 file changed, 6 insertions(+), 172 deletions(-) diff --git a/tests/test_eager_preimports.py b/tests/test_eager_preimports.py index ed62039a..00a2b7da 100644 --- a/tests/test_eager_preimports.py +++ b/tests/test_eager_preimports.py @@ -1,176 +1,13 @@ """ -Tests for `line_profiler.autoprofile.eager_preimports`. +Tests for :py:mod:`line_profiler.autoprofile.eager_preimports`. Notes ----- -Most of the features are already covered by the doctests, but this -project doesn't generally use the `--doctest-modules` option. So this is -mostly a hook to run the doctests. +Most of the features are already covered by the doctests. """ -import doctest -import functools -import importlib -import pathlib -import warnings -import traceback -from types import ModuleType -from typing import Any, Callable, Dict, Type, Union - import pytest -from _pytest import doctest as pytest_doctest - -from line_profiler.autoprofile import eager_preimports, util_static - - -CAN_USE_PYTEST_DOCTEST = True - -try: - class PytestDoctestRunner(pytest_doctest._init_runner_class()): - # Neuter these methods because they expect `out` to be a - # callable while `pytest` passes a list - - def report_start(self, out, *args, **kwargs): - pass - - def report_success(self, out, *args, **kwargs): - pass -except Exception: - CAN_USE_PYTEST_DOCTEST = False - - -def create_doctest_wrapper( - test: doctest.DocTest, *, - fname: Union[str, pathlib.PurePath, None] = None, - globs: Union[Dict[str, Any], None] = None, - name: Union[str, None] = None, - strip_prefix: Union[str, None] = None, - test_name_prefix: str = 'test_doctest_', - use_pytest_doctest: bool = CAN_USE_PYTEST_DOCTEST) -> Callable: - """ - Create a hook to run a doctest as if it was a regular test. - - Returns - wrapper (Callable): - Test function - """ - if strip_prefix is not None: - assert test.name.startswith(strip_prefix) - bare_name = test.name[len(strip_prefix):].lstrip('.').replace('.', '_') - if not bare_name: - bare_name = strip_prefix - else: - bare_name = test.name.replace('.', '_') - if name is None: - name = test_name_prefix + bare_name - assert name.isidentifier() - if fname is not None: - fname = pathlib.Path(fname) - - use_pytest_doctest = bool(use_pytest_doctest) & CAN_USE_PYTEST_DOCTEST - try: - item_from_parent = pytest_doctest.DoctestItem.from_parent - module_from_parent = pytest_doctest.DoctestModule.from_parent - get_doctest_option_flags = pytest_doctest.get_optionflags - xc_to_info = pytest.ExceptionInfo.from_exception - checker = pytest_doctest._get_checker() - except Exception: - use_pytest_doctest = False - def wrapper_pytest(request: pytest.FixtureRequest) -> None: - try: - if globs is not None: - test.globs = globs.copy() - module = module_from_parent(parent=request.session, path=fname) - try: - option_flags = get_doctest_option_flags(request.config) - except AttributeError: - # `pytest < 8` expects an object having the `.config` to - # be passed, instead of the `pytest.Config` itself - option_flags = get_doctest_option_flags(request) - runner = PytestDoctestRunner(checker=checker, - optionflags=option_flags, - continue_on_failure=False) - item = item_from_parent(module, - name=name, runner=runner, dtest=test) - item.setup() - except Exception as e: - # If setting up the test item fails (e.g. due to `pytest` - # refactoring), fall back to the vanilla implementation with - # a warning - tb_lines = traceback.format_exception(type(e), e, e.__traceback__) - msg = ('failed to convert `doctest.DocTest` into ' - f'`pytest.Item`:\n\n{"".join(tb_lines)}\n' - 'falling back to vanilla `doctest`') - warnings.warn(msg) - return wrapper_vanilla() - try: - item.runtest() - except doctest.UnexpectedException as e: - msg = '{}\n{}'.format(e.exc_info[0].__name__, - item.repr_failure(xc_to_info(e))) - raise pytest.fail(msg) from None - - def wrapper_vanilla() -> None: - if globs is not None: - test.globs = globs.copy() - runner = doctest.DebugRunner() - runner.run(test) - - if use_pytest_doctest: - wrapper = wrapper_pytest - else: - wrapper = wrapper_vanilla - - doctest_backend = '_pytest.doctest' if use_pytest_doctest else 'doctest' - wrapper.__name__ = name - wrapper.__doc__ = ('Run the doctest for `{}` with the facilities of `{}`' - .format(bare_name, doctest_backend)) - return wrapper - - -def regularize_doctests( - obj: Any, *, - namespace: Union[Dict[str, Any], None] = None, - finder: Union[doctest.DocTestFinder, None] = None, - strip_common_prefix: bool = True, - use_pytest_doctest: bool = CAN_USE_PYTEST_DOCTEST) -> Dict[str, - Callable]: - """ - Gather doctests from `obj` and make them regular test functions. - - Returns: - wrappers (dict[str, Callable]): - Dictionary from test names to Test functions - """ - if isinstance(obj, ModuleType): - prefix = module = obj.__name__ - fname = obj.__file__ - globs = vars(obj) - else: - module = obj.__module__ - prefix = f'{module}.{obj.__qualname__}' - fname = util_static.modname_to_modpath(module) - globs = vars(importlib.import_module(module)) - - if finder is None: - finder = doctest.DocTestFinder() - - make_wrapper = functools.partial( - create_doctest_wrapper, - fname=fname, globs=globs, strip_prefix=prefix, - use_pytest_doctest=use_pytest_doctest) - - tests = [make_wrapper(test) for test in finder.find(obj) if test.examples] - result = {test.__name__: test for test in tests} - if namespace is None: - return result - for name, test in result.items(): - if name in namespace: - test_module = getattr(namespace.get('__spec__'), 'name', '???') - raise AttributeError(f'module `{test_module}` already has a test ' - f'(or other entity) named `{name}()`') - namespace[name] = test - return result +from line_profiler.autoprofile import eager_preimports @pytest.mark.parametrize( @@ -178,13 +15,10 @@ def regularize_doctests( [('foo; bar', ValueError, None), (1, TypeError, None), ('(foo\n .bar)', ValueError, None)]) -def test_write_eager_import_module_wrong_adder( - adder: Any, xc: Type[Exception], msg: Union[str, None]) -> None: +def test_write_eager_import_module_wrong_adder(adder, xc, msg) -> None: """ - Test passing an erroneous `adder` to `write_eager_import_module()`. + Test passing an erroneous ``adder`` to + :py:meth:`~.write_eager_import_module()`. """ with pytest.raises(xc, match=msg): eager_preimports.write_eager_import_module(['foo'], adder=adder) - - -regularize_doctests(eager_preimports, namespace=globals()) From 3e82eff5bbf05879080e33ca6d1884734636ad97 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 19 May 2025 03:18:44 +0200 Subject: [PATCH 23/70] Refactored `tests/test_explicit_profile.py` tests/test_explicit_profile.py::enter_tmpdir Now explicitly defined as a class, instead of using `@contextlib.contextmanager` (see #340) --- tests/test_explicit_profile.py | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/tests/test_explicit_profile.py b/tests/test_explicit_profile.py index 8cb47506..761529f9 100644 --- a/tests/test_explicit_profile.py +++ b/tests/test_explicit_profile.py @@ -1,20 +1,41 @@ -import contextlib import os import re import sys import tempfile +from contextlib import ExitStack import pytest import ubelt as ub -@contextlib.contextmanager -def enter_tmpdir(): - with contextlib.ExitStack() as stack: - enter = stack.enter_context +class enter_tmpdir: + """ + Set up a temporary directory and :cmd:`chdir` into it. + """ + def __init__(self): + self.stack = ExitStack() + + def __enter__(self): + """ + Returns: + curdir (ubelt.Path) + Temporary directory :cmd:`chdir`-ed into. + + Side effects: + ``curdir`` created and :cmd:`chdir`-ed into. + """ + enter = self.stack.enter_context tmpdir = os.path.abspath(enter(tempfile.TemporaryDirectory())) enter(ub.ChDir(tmpdir)) - yield ub.Path(tmpdir) + return ub.Path(tmpdir) + + def __exit__(self, *_, **__): + """ + Side effects: + * Original working directory restored. + * Temporary directory created deleted. + """ + self.stack.close() def test_simple_explicit_nonglobal_usage(): From 4af36518bd547e7a466e2200c0e0f9887214acdf Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 19 May 2025 03:34:04 +0200 Subject: [PATCH 24/70] Refactored `tests/test_line_profiler.py` tests/test_line_profiler.py::test_profiler_c_callable_no_op() - Updated docstring - Now a parametrized test testing the two no-ops (`.add_callable()` and `.__call__()`) separately --- tests/test_line_profiler.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/test_line_profiler.py b/tests/test_line_profiler.py index 563e1a19..1a1e676a 100644 --- a/tests/test_line_profiler.py +++ b/tests/test_line_profiler.py @@ -506,25 +506,27 @@ def foo(self) -> int: assert profile.enable_count == 0 -def test_profiler_c_callable_no_op(): +@pytest.mark.parametrize('decorate', [True, False]) +def test_profiler_c_callable_no_op(decorate): """ - Test that when the profiler is used to decorate or add a C-level - callable it results in a no-op. + Test that the following are no-ops on C-level callables: + - Decoration (`.__call__()`): the callable is returned as-is. + - `.add_callable()`: it returns 0. """ profile = LineProfiler() - for i, (func, Type) in enumerate([ + for (func, Type) in [ (len, types.BuiltinFunctionType), ('string'.split, types.BuiltinMethodType), (vars(int)['from_bytes'], types.ClassMethodDescriptorType), (str.split, types.MethodDescriptorType), ((1).__str__, types.MethodWrapperType), - (int.__repr__, types.WrapperDescriptorType)]): + (int.__repr__, types.WrapperDescriptorType)]: assert isinstance(func, Type) - if i % 2: # Add is no-op - assert not profile.add_callable(func) - else: # Decoration is no-op + if decorate: # Decoration is no-op assert profile(func) is func + else: # Add is no-op + assert not profile.add_callable(func) assert not profile.functions From 8d2d53468abb0eb1bf7a8ab5378ab48fb373814e Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 19 May 2025 09:52:56 +0200 Subject: [PATCH 25/70] Refactored `tests/test_autoprofile.py` tests/test_autoprofile.py test_autoprofile_exec_{package,module,callable_wrapper_objects}() Refactored tests to simplify call/parametrization signatures --- tests/test_autoprofile.py | 140 ++++++++++++++++++-------------------- 1 file changed, 65 insertions(+), 75 deletions(-) diff --git a/tests/test_autoprofile.py b/tests/test_autoprofile.py index 0aee7354..a0f2bf34 100644 --- a/tests/test_autoprofile.py +++ b/tests/test_autoprofile.py @@ -1,6 +1,7 @@ import os import subprocess import sys +import shlex import tempfile import pytest @@ -432,48 +433,41 @@ def test_autoprofile_script_with_prof_imports(): @pytest.mark.parametrize( - ['use_kernprof_exec', 'prof_mod', 'no_preimports', 'prof_imports', - 'add_one', 'add_two', 'add_operator', 'main'], - [(False, 'test_mod.submod1', False, False, - True, False, True, False), + ('use_kernprof_exec', 'prof_mod', 'flags', 'profiled_funcs'), + [(False, ['test_mod.submod1'], '', {'add_one', 'add_operator'}), # By not using `--no-preimports`, the entirety of `.submod1` is # passed to `add_imported_function_or_module()` - (False, 'test_mod.submod1', True, False, - True, False, False, False), - (False, 'test_mod.submod2', False, True, - False, True, True, False), - (False, 'test_mod', False, True, - True, True, True, True), + (False, ['test_mod.submod1'], '--no-preimports', {'add_one'}), + (False, ['test_mod.submod2'], + '--prof-imports', {'add_two', 'add_operator'}), + (False, ['test_mod'], + '--prof-imports', {'add_one', 'add_two', 'add_operator', '_main'}), # Explicitly add all the modules via multiple `-p` flags, without # using the `--prof-imports` flag - (False, ['test_mod', 'test_mod.submod1,test_mod.submod2'], False, False, - True, True, True, True), - (False, None, False, True, - False, False, False, False), - (True, None, False, True, - False, False, False, False)]) -def test_autoprofile_exec_package( - use_kernprof_exec, prof_mod, no_preimports, prof_imports, - add_one, add_two, add_operator, main): + (False, ['test_mod', 'test_mod.submod1,test_mod.submod2'], + '', {'add_one', 'add_two', 'add_operator', '_main'}), + (False, [], '--prof-imports', set()), + (True, [], '--prof-imports', set())]) +def test_autoprofile_exec_package(use_kernprof_exec, prof_mod, + flags, profiled_funcs): """ Test the execution of a package. """ temp_dpath = ub.Path(tempfile.mkdtemp()) _write_demo_module(temp_dpath) + # Sanity check + all_checked_funcs = {'add_one', 'add_two', 'add_operator', '_main'} + profiled_funcs = set(profiled_funcs) + assert not profiled_funcs - all_checked_funcs + if use_kernprof_exec: args = ['kernprof'] else: args = [sys.executable, '-m', 'kernprof'] - if prof_mod is not None: - if isinstance(prof_mod, str): - prof_mod = [prof_mod] - for pm in prof_mod: - args.extend(['-p', pm]) - if no_preimports: - args.append('--no-preimports') - if prof_imports: - args.append('--prof-imports') + for pm in prof_mod: + args.extend(['-p', pm]) + args.extend(shlex.split(flags)) args.extend(['-l', '-m', 'test_mod', '1', '2', '3']) proc = ub.cmd(args, cwd=temp_dpath, verbose=2) print(proc.stdout) @@ -488,51 +482,48 @@ def test_autoprofile_exec_package( print(raw_output) proc.check_returncode() - assert ('Function: add_one' in raw_output) == add_one - assert ('Function: add_two' in raw_output) == add_two - assert ('Function: add_operator' in raw_output) == add_operator - assert ('Function: _main' in raw_output) == main + for func in all_checked_funcs: + assert (f'Function: {func}' in raw_output) == (func in profiled_funcs) @pytest.mark.parametrize( - ['use_kernprof_exec', 'prof_mod', 'no_preimports', 'prof_imports', - 'add_one', 'add_two', 'add_three', 'add_four', 'add_operator', 'main'], - [(False, 'test_mod.submod2,test_mod.subpkg.submod3.add_three', True, False, - False, True, False, False, False, False), + ('use_kernprof_exec', 'prof_mod', 'flags', 'profiled_funcs'), + [(False, 'test_mod.submod2,test_mod.subpkg.submod3.add_three', + '--no-preimports', {'add_two'}), # By not using `--no-preimports`: # - The entirety of `.submod2` is passed to # `add_imported_function_or_module()` # - Despite not having been imported anywhere, `add_three()` is # still profiled - (False, 'test_mod.submod2,test_mod.subpkg.submod3.add_three', False, False, - False, True, True, False, True, False), - (False, 'test_mod.submod1', False, False, - True, False, False, False, True, False), - (False, 'test_mod.subpkg.submod4', False, True, - True, True, False, True, True, True), - (False, None, False, True, - False, False, False, False, False, False), - (True, None, False, True, - False, False, False, False, False, False)]) -def test_autoprofile_exec_module( - use_kernprof_exec, prof_mod, no_preimports, prof_imports, - add_one, add_two, add_three, add_four, add_operator, main): + (False, 'test_mod.submod2,test_mod.subpkg.submod3.add_three', + '', {'add_two', 'add_three', 'add_operator'}), + (False, 'test_mod.submod1', '', {'add_one', 'add_operator'}), + (False, 'test_mod.subpkg.submod4', + '--prof-imports', + {'add_one', 'add_two', 'add_four', 'add_operator', '_main'}), + (False, None, '--prof-imports', {}), + (True, None, '--prof-imports', {})]) +def test_autoprofile_exec_module(use_kernprof_exec, prof_mod, + flags, profiled_funcs): """ Test the execution of a module. """ temp_dpath = ub.Path(tempfile.mkdtemp()) _write_demo_module(temp_dpath) + # Sanity check + all_checked_funcs = {'add_one', 'add_two', 'add_three', 'add_four', + 'add_operator', '_main'} + profiled_funcs = set(profiled_funcs) + assert not profiled_funcs - all_checked_funcs + if use_kernprof_exec: args = ['kernprof'] else: args = [sys.executable, '-m', 'kernprof'] if prof_mod is not None: args.extend(['-p', prof_mod]) - if no_preimports: - args.append('--no-preimports') - if prof_imports: - args.append('--prof-imports') + args.extend(shlex.split(flags)) args.extend(['-l', '-m', 'test_mod.subpkg.submod4', '1', '2', '3']) proc = ub.cmd(args, cwd=temp_dpath, verbose=2) print(proc.stdout) @@ -547,12 +538,8 @@ def test_autoprofile_exec_module( print(raw_output) proc.check_returncode() - assert ('Function: add_one' in raw_output) == add_one - assert ('Function: add_two' in raw_output) == add_two - assert ('Function: add_three' in raw_output) == add_three - assert ('Function: add_four' in raw_output) == add_four - assert ('Function: add_operator' in raw_output) == add_operator - assert ('Function: _main' in raw_output) == main + for func in all_checked_funcs: + assert (f'Function: {func}' in raw_output) == (func in profiled_funcs) @pytest.mark.parametrize('view', [True, False]) @@ -651,15 +638,14 @@ def test_autoprofile_from_inlined_script(outfile, expected_outfile) -> None: @pytest.mark.parametrize( - ('prof_mod, function, method, class_method, static_method, ' - 'descriptor'), - [('my_module', True, True, True, True, True), + ('prof_mod', 'profiled_funcs'), + [('my_module', + {'function', 'method', 'class_method', 'static_method', 'descriptor'}), # `function()` included in profiling via `Class.partial_method()` - ('my_module.Class', True, True, True, True, True), - ('my_module.Class.descriptor', False, False, False, False, True)]) -def test_autoprofile_callable_wrapper_objects( - prof_mod, function, method, class_method, static_method, - descriptor): + ('my_module.Class', + {'function', 'method', 'class_method', 'static_method', 'descriptor'}), + ('my_module.Class.descriptor', {'descriptor'})]) +def test_autoprofile_callable_wrapper_objects(prof_mod, profiled_funcs): """ Test that on-import profiling catches various callable-wrapper object types: @@ -669,6 +655,16 @@ def test_autoprofile_callable_wrapper_objects( - partialmethod Like it does regular methods and functions. """ + # Sanity check + all_checked_funcs = {'function', 'method', + 'partial_method', 'class_method', 'static_method', + 'descriptor'} + profiled_funcs = set(profiled_funcs) + assert not profiled_funcs - all_checked_funcs + # Note: `partial_method()` not to be included as its own item + # because it's a wrapper around `function()` + assert 'partial_method' not in profiled_funcs + with tempfile.TemporaryDirectory() as tmpdir: temp_dpath = ub.Path(tmpdir) path = temp_dpath / 'path' @@ -723,11 +719,5 @@ def descriptor(self): print(proc.stderr) proc.check_returncode() - assert ('Function: function' in raw_output) == function - assert ('Function: method' in raw_output) == method - assert ('Function: class_method' in raw_output) == class_method - assert ('Function: static_method' in raw_output) == static_method - # `partial_method()` not included as its own item because it's a - # wrapper around `function()` - assert 'Function: partial_method' not in raw_output - assert ('Function: descriptor' in raw_output) == descriptor + for func in all_checked_funcs: + assert (f'Function: {func}' in raw_output) == (func in profiled_funcs) From 70ce3acdcbb7efc69c5a35629eabc1a4ea24591b Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 19 May 2025 18:58:19 +0200 Subject: [PATCH 26/70] `kernprof.py` doc updates and refactoring kernprof.py __doc__ - Added `sphinx` roles to generic inlined code (e.g. ':command:', ':option:') - Rephrased documentation for the `-l` flag execfile.__doc__ RepeatedTimer.__doc__ find_script.__doc__ _python_command.__doc__ _normalize_profiling_targets.__doc__ _restore_list.__doc__ pre_parse_single_arg_directive.__doc__ _write_tempfile.__doc__ _main.__doc__ Added `sphinx` roles to generic inlined code (e.g. ':command:', ':option:') _write_preimports() Refactored from big chunk of code in `_main()` handling the pre-imports --- kernprof.py | 219 ++++++++++++++++++++++++++++------------------------ 1 file changed, 120 insertions(+), 99 deletions(-) diff --git a/kernprof.py b/kernprof.py index 46a5f636..fb086df3 100755 --- a/kernprof.py +++ b/kernprof.py @@ -1,8 +1,10 @@ #!/usr/bin/env python """ -Script to conveniently run profilers on code in a variety of circumstances. +Script to conveniently run profilers on code in a variety of +circumstances. -To profile a script, decorate the functions of interest with ``@profile`` +To profile a script, decorate the functions of interest with +:py:deco:`profile`: .. code:: bash @@ -15,25 +17,27 @@ def main(): NOTE: - New in 4.1.0: Instead of relying on injecting ``profile`` into the builtins - you can now ``import line_profiler`` and use ``line_profiler.profile`` to - decorate your functions. This allows the script to remain functional even - if it is not actively profiled. See :py:mod:`line_profiler` for details. + New in 4.1.0: Instead of relying on injecting :py:deco:`profile` + into the builtins you can now ``import line_profiler`` and use + :py:deco:`line_profiler.profile` to decorate your functions. This + allows the script to remain functional even if it is not actively + profiled. See :py:mod:`line_profiler` for details. -Then run the script using kernprof: +Then run the script using :command:`kernprof`: .. code:: bash kernprof -b script_to_profile.py -By default this runs with the default :py:mod:`cProfile` profiler and does not -require compiled modules. Instructions to view the results will be given in the -output. Alternatively, adding ``-v`` to the command line will write results to -stdout. +By default this runs with the default :py:mod:`cProfile` profiler and +does not require compiled modules. Instructions to view the results will +be given in the output. Alternatively, adding :option:`-v` to the +command line will write results to stdout. -To enable line-by-line profiling, then :py:mod:`line_profiler` must be -available and compiled. Add the ``-l`` argument to the kernprof invocation. +To enable line-by-line profiling, :py:mod:`line_profiler` must be +available and compiled, and the :option:`-l` argument should be added to +the :command:`kernprof` invocation: .. code:: bash @@ -43,17 +47,21 @@ def main(): New in 4.3.0: More code execution options are added: - * ``kernprof -m some.module `` parallels - ``python -m`` and runs the provided module as ``__main__``. - * ``kernprof -c "some code" `` parallels - ``python -c`` and executes the provided literal code. - * ``kernprof - `` parallels ``python -`` and - executes literal code passed via the ``stdin``. + * :command:`kernprof -m some.module ` + parallels :command:`python -m` and runs the provided module as + ``__main__``. + * :command:`kernprof -c "some code" ` + parallels :command:`python -c` and executes the provided literal + code. + * :command:`kernprof - ` parallels + :command:`python -` and executes literal code passed via the + ``stdin``. - See also :doc:`kernprof invocations `. + See also + :doc:`kernprof invocations `. For more details and options, refer to the CLI help. -To view kernprof help run: +To view :command:`kernprof` help run: .. code:: bash @@ -98,12 +106,13 @@ def main(): NOTE: New in 4.3.0: For more intuitive profiling behavior, profiling - targets in ``--prof-mod`` (except the profiled script/code) are now - eagerly pre-imported to be profiled + targets in :option:`--prof-mod` (except the profiled script/code) + are now eagerly pre-imported to be profiled (see :py:mod:`line_profiler.autoprofile.eager_preimports`), regardless of whether those imports directly occur in the profiled script/module/code. - To restore the old behavior, pass the ``--no-preimports`` flag. + To restore the old behavior, pass the :option:`--no-preimports` + flag. """ import builtins import functools @@ -134,15 +143,15 @@ def main(): def execfile(filename, globals=None, locals=None): - """ Python 3.x doesn't have 'execfile' builtin """ + """ Python 3.x doesn't have :py:func:`execfile` builtin """ with open(filename, 'rb') as f: exec(compile(f.read(), filename, 'exec'), globals, locals) # ===================================== class ContextualProfile(ByCountProfilerMixin, Profile): - """ A subclass of Profile that adds a context manager for Python - 2.5 with: statements and a decorator. + """ A subclass of :py:class:`Profile` that adds a context manager + for Python 2.5 with: statements and a decorator. """ def __init__(self, *args, **kwds): super(ByCountProfilerMixin, self).__init__(*args, **kwds) @@ -174,7 +183,7 @@ def disable_by_count(self): class RepeatedTimer: """ - Background timer for outputting file every n seconds. + Background timer for outputting file every ``n`` seconds. Adapted from [SO474528]_. @@ -223,7 +232,7 @@ def find_module_script(module_name): def find_script(script_name, exit_on_error=True): """ Find the script. - If the input is not a file, then $PATH will be searched. + If the input is not a file, then :envvar:`PATH` will be searched. """ if os.path.isfile(script_name): return script_name @@ -245,7 +254,7 @@ def find_script(script_name, exit_on_error=True): def _python_command(): """ - Return a command that corresponds to :py:obj:`sys.executable`. + Return a command that corresponds to :py:data:`sys.executable`. """ import shutil for abbr in 'python', 'python3': @@ -256,7 +265,7 @@ def _python_command(): def _normalize_profiling_targets(targets): """ - Normalize the parsed ``--prof-mod`` by: + Normalize the parsed :option:`--prof-mod` by: * Normalizing file paths with :py:func:`find_script()`, and subsequently to absolute paths. @@ -285,7 +294,7 @@ def find(path): class _restore_list: """ - Restore a list like ``sys.path`` after running code which + Restore a list like :py:data:`sys.path` after running code which potentially modifies it. Example @@ -329,7 +338,8 @@ def wrapper(*args, **kwargs): def pre_parse_single_arg_directive(args, flag, sep='--'): """ Pre-parse high-priority single-argument directives like - ``-m module`` to emulate the behavior of ``python [...]``. + :option:`-m module` to emulate the behavior of + :command:`python [...]`. Examples -------- @@ -521,7 +531,8 @@ def positive_float(value): def _write_tempfile(source, content, options, tmpdir): """ - Called by ``main()`` to handle ``kernprof -c`` and ``kernprof -``; + Called by :py:func:`main()` to handle :command:`kernprof -c` and + :command:`kernprof -`; not to be invoked on its own. """ import textwrap @@ -548,9 +559,81 @@ def _write_tempfile(source, content, options, tmpdir): suffix='.' + extension) +def _write_preimports(prof, prof_mod, exclude): + """ + Called by :py:func:`main()` to handle eager pre-imports; + not to be invoked on its own. + """ + from line_profiler.autoprofile.eager_preimports import ( + is_dotted_path, propose_names, write_eager_import_module) + from line_profiler.autoprofile.util_static import modpath_to_modname + from line_profiler.autoprofile.autoprofile import ( + _extend_line_profiler_for_profiling_imports as upgrade_profiler) + + filtered_targets = [] + invalid_targets = [] + for target in prof_mod: + if is_dotted_path(target): + filtered_targets.append(target) + continue + # Paths already normalized by `_normalize_profiling_targets()` + if not os.path.exists(target): + invalid_targets.append(target) + continue + if any(os.path.samefile(target, excluded) for excluded in exclude): + # Ignore the script to be run in eager importing + # (`line_profiler.autoprofile.autoprofile.run()` will handle + # it) + continue + modname = modpath_to_modname(target) + if modname is None: # Not import-able + invalid_targets.append(target) + continue + filtered_targets.append(modname) + if invalid_targets: + invalid_targets = sorted(set(invalid_targets)) + msg = ('{} profile-on-import target{} cannot be converted to ' + 'dotted-path form: {!r}' + .format(len(invalid_targets), + '' if len(invalid_targets) == 1 else 's', + invalid_targets)) + warnings.warn(msg) + if not filtered_targets: + return + # - We could've done everything in-memory with `io.StringIO` and + # `exec()`, but that results in indecipherable tracebacks should + # anything goes wrong; + # so we write to a tempfile and `execfile()` it + # - While this works theoretically for preserving traceback, the + # catch is that the tempfile will already have been deleted by the + # time the traceback is formatted; + # so we have to format the traceback and manually print the + # context before re-raising the error + upgrade_profiler(prof) + temp_mod_name = next( + name for name in propose_names(['_kernprof_eager_preimports']) + if name not in sys.modules) + with tempfile.TemporaryDirectory() as tmpdir: + temp_mod_path = os.path.join(tmpdir, temp_mod_name + '.py') + with open(temp_mod_path, mode='w') as fobj: + write_eager_import_module(filtered_targets, stream=fobj) + ns = {} # Use a fresh namespace + try: + execfile(temp_mod_path, ns, ns) + except Exception as e: + tb_lines = traceback.format_tb(e.__traceback__) + i_last_temp_frame = max( + i for i, line in enumerate(tb_lines) + if temp_mod_path in line) + print('\nContext:', ''.join(tb_lines[i_last_temp_frame:]), + end='', sep='\n', file=sys.stderr) + raise + + def _main(options, module=False): """ - Called by ``main()`` for the actual execution and profiling of code; + Called by :py:func:`main()` for the actual execution and profiling + of code; not to be invoked on its own. """ if not options.outfile: @@ -626,70 +709,8 @@ def _main(options, module=False): # contains the script file to be run. E.g. the script may not # even have a `if __name__ == '__main__': ...` guard. So don't # eager-import it. - from line_profiler.autoprofile.eager_preimports import ( - is_dotted_path, propose_names, write_eager_import_module) - from line_profiler.autoprofile.util_static import modpath_to_modname - from line_profiler.autoprofile.autoprofile import ( - _extend_line_profiler_for_profiling_imports as upgrade_profiler) - - filtered_targets = [] - invalid_targets = [] - for target in options.prof_mod: - if is_dotted_path(target): - filtered_targets.append(target) - continue - # Filenames are already normalized in - # `_normalize_profiling_targets()` - if not os.path.exists(target): - invalid_targets.append(target) - continue - if not module and os.path.samefile(target, script_file): - # Ignore the script to be run in eager importing - # (`line_profiler.autoprofile.autoprofile.run()` will - # handle it) - continue - modname = modpath_to_modname(target) - if modname is None: # Not import-able - invalid_targets.append(target) - continue - filtered_targets.append(modname) - if invalid_targets: - invalid_targets = sorted(set(invalid_targets)) - msg = ('{} profile-on-import target{} cannot be converted to ' - 'dotted-path form: {!r}' - .format(len(invalid_targets), - '' if len(invalid_targets) == 1 else 's', - invalid_targets)) - warnings.warn(msg) - if filtered_targets: - # - We could've done everything in-memory with `io.StringIO` - # and `exec()`, but that results in indecipherable - # tracebacks should anything goes wrong; - # so we write to a tempfile and `execfile()` it - # - While this works theoretically for preserving traceback, - # the catch is that the tempfile will already have been - # deleted by the time the traceback is formatted; - # so we have to format the traceback and manually print - # the context before re-raising the error - upgrade_profiler(prof) - temp_mod_name = next( - name for name in propose_names(['_kernprof_eager_preimports']) - if name not in sys.modules) - with tempfile.TemporaryDirectory() as tmpdir: - temp_mod_path = os.path.join(tmpdir, temp_mod_name + '.py') - with open(temp_mod_path, mode='w') as fobj: - write_eager_import_module(filtered_targets, stream=fobj) - ns = {} # Use a fresh namespace - try: - execfile(temp_mod_path, ns, ns) - except Exception as e: - tb_lines = traceback.format_tb(e.__traceback__) - i_last_temp_frame = max( - i for i, line in enumerate(tb_lines) - if temp_mod_path in line) - print('\nContext:', ''.join(tb_lines[i_last_temp_frame:]), - end='', sep='\n', file=sys.stderr) - raise + exclude = set() if module else {script_file} + _write_preimports(prof, options.prof_mod, exclude) if options.output_interval: rt = RepeatedTimer(max(options.output_interval, 1), prof.dump_stats, options.outfile) From 48d3983857b3720aadfe67a16c3a282cbeeabc1a Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 19 May 2025 23:46:52 +0200 Subject: [PATCH 27/70] Doc updates kernprof.py - Fixed broken link to `@line_profiler.profile` (or `line_profiler.explicit_profiler.GlobalProfiler`) - Suppressed link for the `-v` flag which linked to some nonsense line_profiler/__init__.py - Added anchor to section `Line Profiler Basic Usage` - Replaced inlines with the appropriate roles - Fixed broken link to `@line_profiler.profile` - Fixed accidentally uncommented TODO --- kernprof.py | 32 +++++++++++++++++--------------- line_profiler/__init__.py | 23 ++++++++++++++--------- 2 files changed, 31 insertions(+), 24 deletions(-) diff --git a/kernprof.py b/kernprof.py index fb086df3..118fdd46 100755 --- a/kernprof.py +++ b/kernprof.py @@ -4,7 +4,7 @@ circumstances. To profile a script, decorate the functions of interest with -:py:deco:`profile`: +:py:deco:`profile `: .. code:: bash @@ -19,12 +19,14 @@ def main(): New in 4.1.0: Instead of relying on injecting :py:deco:`profile` into the builtins you can now ``import line_profiler`` and use - :py:deco:`line_profiler.profile` to decorate your functions. This - allows the script to remain functional even if it is not actively - profiled. See :py:mod:`line_profiler` for details. + :py:deco:`line_profiler.profile ` + to decorate your functions. This allows the script to remain + functional even if it is not actively profiled. See + :py:mod:`!line_profiler` (:ref:`link `) for + details. -Then run the script using :command:`kernprof`: +Then run the script using :program:`kernprof`: .. code:: bash @@ -32,12 +34,12 @@ def main(): By default this runs with the default :py:mod:`cProfile` profiler and does not require compiled modules. Instructions to view the results will -be given in the output. Alternatively, adding :option:`-v` to the +be given in the output. Alternatively, adding :option:`!-v` to the command line will write results to stdout. To enable line-by-line profiling, :py:mod:`line_profiler` must be -available and compiled, and the :option:`-l` argument should be added to -the :command:`kernprof` invocation: +available and compiled, and the :option:`!-l` argument should be added to +the :program:`kernprof` invocation: .. code:: bash @@ -49,19 +51,19 @@ def main(): * :command:`kernprof -m some.module ` parallels :command:`python -m` and runs the provided module as - ``__main__``. + :py:mod:`__main__`. * :command:`kernprof -c "some code" ` parallels :command:`python -c` and executes the provided literal code. * :command:`kernprof - ` parallels :command:`python -` and executes literal code passed via the - ``stdin``. + :file:`stdin`. See also :doc:`kernprof invocations `. For more details and options, refer to the CLI help. -To view :command:`kernprof` help run: +To view the :program:`kernprof` help text run: .. code:: bash @@ -106,12 +108,12 @@ def main(): NOTE: New in 4.3.0: For more intuitive profiling behavior, profiling - targets in :option:`--prof-mod` (except the profiled script/code) + targets in :option:`!--prof-mod` (except the profiled script/code) are now eagerly pre-imported to be profiled (see :py:mod:`line_profiler.autoprofile.eager_preimports`), regardless of whether those imports directly occur in the profiled script/module/code. - To restore the old behavior, pass the :option:`--no-preimports` + To restore the old behavior, pass the :option:`!--no-preimports` flag. """ import builtins @@ -265,7 +267,7 @@ def _python_command(): def _normalize_profiling_targets(targets): """ - Normalize the parsed :option:`--prof-mod` by: + Normalize the parsed :option:`!--prof-mod` by: * Normalizing file paths with :py:func:`find_script()`, and subsequently to absolute paths. @@ -338,7 +340,7 @@ def wrapper(*args, **kwargs): def pre_parse_single_arg_directive(args, flag, sep='--'): """ Pre-parse high-priority single-argument directives like - :option:`-m module` to emulate the behavior of + :option:`!-m module` to emulate the behavior of :command:`python [...]`. Examples diff --git a/line_profiler/__init__.py b/line_profiler/__init__.py index 20a640e3..4259070e 100644 --- a/line_profiler/__init__.py +++ b/line_profiler/__init__.py @@ -33,11 +33,13 @@ pip install line_profiler[all] +.. _line-profiler-basic-usage: + Line Profiler Basic Usage ========================= To demonstrate line profiling, we first need to generate a Python script to -profile. Write the following code to a file called ``demo_primes.py``. +profile. Write the following code to a file called :file:`demo_primes.py`: .. code:: python @@ -80,8 +82,10 @@ def main(): main() -In this script we explicitly import the ``profile`` function from -``line_profiler``, and then we decorate function of interest with ``@profile``. +In this script we explicitly import the +:py:deco:`profile ` function +from :py:mod:`line_profiler`, and then we decorate function of interest with +:py:deco:`profile`. By default nothing is profiled when running the script. @@ -99,17 +103,18 @@ def main(): The quickest way to enable profiling is to set the environment variable -``LINE_PROFILE=1`` and running your script as normal. - -.... todo: add a link that points to docs showing all the different ways to enable profiling. +:envvar:`LINE_PROFILE=1 ` and running your script as normal. +.. .. todo: add a link that points to docs showing all the different ways to +.. .. to enable profiling. .. code:: bash LINE_PROFILE=1 python demo_primes.py -This will output 3 files: profile_output.txt, profile_output_.txt, -and profile_output.lprof and stdout will look something like: +This will output 3 files: :file:`profile_output.txt`, +:file:`profile_output_.txt`, and :file:`profile_output.lprof`; and +:file:`stdout` will look something like: .. code:: @@ -219,7 +224,7 @@ def main(): * `timeit `_: The builtin timeit module for profiling single statements. -* `timerit `_: A multi-statements alternative to the builtin ``timeit`` module. +* `timerit `_: A multi-statements alternative to the builtin :py:mod:`timeit` module. * `torch.profiler `_ tools for profiling torch code. From 9d2e0ada1d99a51f77419919dfd8c3c93ec6adb4 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 24 May 2025 06:31:52 +0200 Subject: [PATCH 28/70] Refactored scoping policies line_profiler/autoprofile/line_profiler_utils.py[i] add_imported_function_or_module() Updated documentation and type annotation on parameter `scoping_policy` line_profiler/line_profiler.py[i] ScopingPolicy __doc__ Updated EXACT Changed semantics (previous `.EXACT` is now `.CHILDREN`) to disable descension into classes and modules CHILDREN New enum value (approximate equal to the previous `.EXACT`) __init_subclass__(), _check_class() Extended checks get_filter() - New method for getting filter functions for various namespace and member types - Added handling for function and module members to_policies() New class method for normalizing strings and mappings into mapping of policies LineProfiler __call__(), add_callable(), add_module(), add_class() Extended docstrings add_callable() Added parameter `guard` for controlling what functions to pass to `.add_function()` add_module(), add_class() - Updated type annotations on parameter `scoping_policy` - Function profiling now controlled by `scoping_policy` (previous behavior equivalent to `scoping_policy='none'`) add_module() Descension into modules now contolled by `scoping_policy` (previous behavior equivalent to `scoping_policy='exact'`) tests/test_explicit_profile.py::test_profiler_class_scope_matching() Refactored from `test_profiler_scope_matching()` TODO: add scoping tests for modules and functions --- .../autoprofile/line_profiler_utils.py | 4 + .../autoprofile/line_profiler_utils.pyi | 13 +- line_profiler/line_profiler.py | 382 +++++++++++++----- line_profiler/line_profiler.pyi | 56 ++- tests/test_explicit_profile.py | 26 +- 5 files changed, 368 insertions(+), 113 deletions(-) diff --git a/line_profiler/autoprofile/line_profiler_utils.py b/line_profiler/autoprofile/line_profiler_utils.py index 355af7ee..5f2984b7 100644 --- a/line_profiler/autoprofile/line_profiler_utils.py +++ b/line_profiler/autoprofile/line_profiler_utils.py @@ -25,6 +25,10 @@ def add_imported_function_or_module( details. Strings are converted to :py:class:`~.ScopingPolicy` instances in a case-insensitive manner. + Can also be a mapping from the keys ``'func'``, ``'class'``, + and ``'module'`` to :py:class:`~.ScopingPolicy` objects or + strings convertible thereto, in which case different + policies can be enacted for these object types. wrap (bool): Whether to replace the wrapped members with wrappers which automatically enable/disable the profiler when called. diff --git a/line_profiler/autoprofile/line_profiler_utils.pyi b/line_profiler/autoprofile/line_profiler_utils.pyi index 0d859824..847737fa 100644 --- a/line_profiler/autoprofile/line_profiler_utils.pyi +++ b/line_profiler/autoprofile/line_profiler_utils.pyi @@ -2,15 +2,16 @@ from types import ModuleType from typing import overload, Any, Literal, TYPE_CHECKING if TYPE_CHECKING: # Stub-only annotations - from ..line_profiler import ( - CLevelCallable, CallableLike, ScopingPolicy, - ) + from ..line_profiler import (CLevelCallable, CallableLike, + ScopingPolicy, ScopingPolicyDict) @overload def add_imported_function_or_module( self, item: CLevelCallable | Any, - scoping_policy: ScopingPolicy | str = ScopingPolicy.SIBLINGS, + scoping_policy: (ScopingPolicy + | str + | ScopingPolicyDict) = ScopingPolicy.SIBLINGS, wrap: bool = False) -> Literal[0]: ... @@ -18,6 +19,8 @@ def add_imported_function_or_module( @overload def add_imported_function_or_module( self, item: CallableLike | type | ModuleType, - scoping_policy: ScopingPolicy | str = ScopingPolicy.SIBLINGS, + scoping_policy: (ScopingPolicy + | str + | ScopingPolicyDict) = ScopingPolicy.SIBLINGS, wrap: bool = False) -> Literal[0, 1]: ... diff --git a/line_profiler/line_profiler.py b/line_profiler/line_profiler.py index e44ce565..f4feb18d 100755 --- a/line_profiler/line_profiler.py +++ b/line_profiler/line_profiler.py @@ -15,6 +15,7 @@ import warnings from argparse import ArgumentError, ArgumentParser from enum import auto +from typing import Union, TypedDict try: from ._line_profiler import LineProfiler as CLineProfiler @@ -109,34 +110,77 @@ def __init__(self, func, profiler_id): class ScopingPolicy(StringEnum): """ - :py:class:`StrEnum` for scoping policies, that is, how nested - namespaces (classes and modules) are descended into when using - :py:meth:`LineProfiler.add_class`, + :py:class:`StrEnum` for scoping policies, that is, how it is + decided whether to: + + * Profile a function found in a namespace (a class or a module), and + * Descend into nested namespaces so that their methods and functions + are profiled, + + when using :py:meth:`LineProfiler.add_class`, :py:meth:`LineProfiler.add_module`, and :py:func:`~.add_imported_function_or_module()`. Available policies are: :py:attr:`ScopingPolicy.EXACT` - Only add classes defined locally in the very module, or - the very class as its "inner classes" + Only profile *functions* found in the namespace fulfilling + :py:attr:`ScopingPolicy.CHILDREN` as defined below, without + descending into nested namespaces + + :py:attr:`ScopingPolicy.CHILDREN` + Only profile/descend into *child* objects, which are: + + * Classes and functions defined *locally* in the very + module, or in the very class as its "inner classes" and + methods + * Direct submodules, in case when the namespace is a module + object representing a package + :py:attr:`ScopingPolicy.DESCENDANTS` - Only add locally-defined classes (see :py:attr:`EXACT`), - their locally-defined classes, and so on + Only profile/descend into *descendant* objects, which are: + + * Child classes, functions, and modules, as defined above in + :py:attr:`ScopingPolicy.CHILDREN` + * Their child classes, functions, and modules, ... + * ... and so on + :py:attr:`ScopingPolicy.SIBLINGS` - Only add classes fulfilling :py:attr:`DESCENDANTS`, or are - defined in the same module as this very class, or are - defined in sibling modules and subpackages (if a part of a - package) to this very module + Only profile/descend into *sibling* and descendant objects, + which are: + + * Descendant classes, functions, and modules, as defined above + in :py:attr:`ScopingPolicy.DESCENDANTS` + * Classes and functions (and descendants thereof) defined in the + same parent namespace to this very class, or in modules (and + subpackages and their descendants) sharing a parent package + to this very module + * Modules (and subpackages and their descendants) sharing a + parent package, when the namespace is a module + :py:attr:`ScopingPolicy.NONE` - Don't check scopes and add all classes in the local - namespace of the class/module + Don't check scopes; profile all functions found in the local + namespace of the class/module, and descend into all nested + namespaces recursively + + Note: + This is probably a very bad idea for module scoping; + proceed with care. + + Note: + Other than :py:class:`enum.Enum` methods starting and ending + with single underscores (e.g. :py:meth:`!_missing_`), all + methods prefixed with a single underscore are to be considered + implementation details. """ EXACT = auto() + CHILDREN = auto() DESCENDANTS = auto() SIBLINGS = auto() NONE = auto() + # Verification + def __init_subclass__(cls, *args, **kwargs): """ Call :py:meth:`_check_class`. @@ -147,9 +191,8 @@ def __init_subclass__(cls, *args, **kwargs): @classmethod def _check_class(cls): """ - Verify that :py:meth:`_add_module_filter` and - :py:meth:`_add_class_filter` returns a callable for all policy - values. + Verify that :py:meth:`.get_filter` return a callable for all + policy values and object types. """ mock_module = types.ModuleType('mock_module') @@ -157,63 +200,118 @@ class MockClass: pass for member in cls.__members__.values(): - assert callable(member._add_module_filter(mock_module)) - assert callable(member._add_class_filter(MockClass)) + for obj_type in 'func', 'class', 'module': + for namespace in mock_module, MockClass: + assert callable(member.get_filter(namespace, obj_type)) - @staticmethod - def _no_op(_): - """ - Filter that is always true. - """ - return True + # Filtering - def _add_module_filter(self, mod): + def get_filter(self, namespace, obj_type): """ Args: - mod (ModuleType): - Module to be profiled. + namespace (Union[type, types.ModuleType]): + Class or module to be profiled. + obj_type (Literal['func', 'class', 'module']): + Type of object encountered in ``namespace``: - Returns: - func (Callable[[type], bool]): - Filter callable returning whether the argument, a class - in the local namespace of ``mod``, should be descended - into and added via :py:meth:`LineProfiler.add_class` - """ - def match_prefix(s, prefix, sep='.'): - return s == prefix or s.startswith(prefix + sep) - - def class_is_child(other): - return other.__module__ == mod.__name__ + ``'func'`` + Either a function, or a component function of a + callable-like object (e.g. :py:class:`property`) - def class_is_descendant(other): - return match_prefix(other.__module__, mod.__name__) + ``'class'`` (resp. ``'module'``) + A class (resp. a module) - def class_is_cousin(other): - if class_is_descendant(other): - return True - return match_prefix(other.__module__, parent) - - parent, _, basename = mod.__name__.rpartition('.') - return {'exact': class_is_child, - 'descendants': class_is_descendant, - 'siblings': (class_is_cousin # Only if a pkg - if basename else - class_is_descendant), - 'none': self._no_op}[self.value] + Returns: + func (Callable[..., bool]): + Filter callable returning whether the argument (as + specified by ``obj_type``) should be added + via :py:meth:`LineProfiler.add_class`, + :py:meth:`LineProfiler.add_module`, or + :py:meth:`LineProfiler.add_callable` + """ + is_class = isinstance(namespace, type) + if obj_type == 'module': + if is_class: + return self._return_const(False) + return self._get_module_filter_in_module(namespace) + if is_class: + method = self._get_callable_filter_in_class + else: + method = self._get_callable_filter_in_module + return method(namespace, is_class=(obj_type == 'class')) - def _add_class_filter(self, cls): + @classmethod + def to_policies(cls, policies): """ + Normalize ``policies`` into a dictionary of policies for various + object types. + Args: - cls (type): - Class to be profiled. + policies (Union[str, ScopingPolicy, ScopingPolicyDict]): + :py:class:`ScopingPolicy`, string convertible thereto + (case-insensitive), or a mapping containing such values + and the keys as outlined in the return value Returns: - func (Callable[[type], bool]): - Filter callable returning whether the argument, a class - in the local namespace of ``cls``, should be descended - into and added via :py:meth:`LineProfiler.add_class` + normalized_policies (dict[str, ScopingPolicy]): + Dictionary with the following key-value pairs: + + ``'func'`` + :py:class:`ScopingPolicy` for profiling functions + and other callable-like objects composed thereof + (e.g. :py:class:`property`). + + ``'class'`` + :py:class:`ScopingPolicy` for descending into + classes. + + ``'module'`` + :py:class:`ScopingPolicy` for descending into + modules (if the namespace is itself a module). + + Note: + If ``policies`` is a mapping, it is required to contain all + three of the aforementioned keys. + + Example: + + >>> assert (ScopingPolicy.to_policies('children') + ... == dict.fromkeys(['func', 'class', 'module'], + ... ScopingPolicy.CHILDREN)) + >>> assert (ScopingPolicy.to_policies({ + ... 'func': 'NONE', + ... 'class': 'descendants', + ... 'module': 'exact', + ... 'unused key': 'unused value'}) + ... == {'func': ScopingPolicy.NONE, + ... 'class': ScopingPolicy.DESCENDANTS, + ... 'module': ScopingPolicy.EXACT}) + >>> ScopingPolicy.to_policies({}) + Traceback (most recent call last): + ... + KeyError: 'func' """ - def class_is_child(other): + if isinstance(policies, str): + policy = cls(policies) + return _ScopingPolicyDict( + dict.fromkeys(['func', 'class', 'module'], policy)) + return _ScopingPolicyDict({'func': cls(policies['func']), + 'class': cls(policies['class']), + 'module': cls(policies['module'])}) + + @staticmethod + def _return_const(value): + def return_const(*_, **__): + return value + + return return_const + + @staticmethod + def _match_prefix(s, prefix, sep='.'): + return s == prefix or s.startswith(prefix + sep) + + def _get_callable_filter_in_class(self, cls, is_class): + def func_is_child(other): if not modules_are_equal(other): return False return other.__qualname__ == f'{cls.__qualname__}.{other.__name__}' @@ -221,21 +319,75 @@ def class_is_child(other): def modules_are_equal(other): # = sibling check return cls.__module__ == other.__module__ - def class_is_descendant(other): + def func_is_descdendant(other): if not modules_are_equal(other): return False return other.__qualname__.startswith(cls.__qualname__ + '.') - return {'exact': class_is_child, - 'descendants': class_is_descendant, + return {'exact': (self._return_const(False) + if is_class else + func_is_child), + 'children': func_is_child, + 'descendants': func_is_descdendant, 'siblings': modules_are_equal, - 'none': self._no_op}[self.value] + 'none': self._return_const(True)}[self.value] + + def _get_callable_filter_in_module(self, mod, is_class): + def func_is_child(other): + return other.__module__ == mod.__name__ + + def func_is_descdendant(other): + return self._match_prefix(other.__module__, mod.__name__) + + def func_is_cousin(other): + if func_is_descdendant(other): + return True + return self._match_prefix(other.__module__, parent) + + parent, _, basename = mod.__name__.rpartition('.') + return {'exact': (self._return_const(False) + if is_class else + func_is_child), + 'children': func_is_child, + 'descendants': func_is_descdendant, + 'siblings': (func_is_cousin # Only if a pkg + if basename else + func_is_descdendant), + 'none': self._return_const(True)}[self.value] + + def _get_module_filter_in_module(self, mod): + def module_is_descendant(other): + return other.__name__.startswith(mod.__name__ + '.') + + def module_is_child(other): + return other.__name__.rpartition('.')[0] == mod.__name__ + + def module_is_sibling(other): + return other.__name__.startswith(parent + '.') + + parent, _, basename = mod.__name__.rpartition('.') + return {'exact': self._return_const(False), + 'children': module_is_child, + 'descendants': module_is_descendant, + 'siblings': (module_is_sibling # Only if a pkg + if basename else + self._return_const(False)), + 'none': self._return_const(True)}[self.value] # Sanity check in case we extended `ScopingPolicy` and forgot to update # the corresponding methods ScopingPolicy._check_class() +ScopingPolicyDict = TypedDict('ScopingPolicyDict', + {'func': Union[str, ScopingPolicy], + 'class': Union[str, ScopingPolicy], + 'module': Union[str, ScopingPolicy]}) +_ScopingPolicyDict = TypedDict('_ScopingPolicyDict', + {'func': ScopingPolicy, + 'class': ScopingPolicy, + 'module': ScopingPolicy}) + class LineProfiler(CLineProfiler, ByCountProfilerMixin): """ @@ -256,9 +408,9 @@ class LineProfiler(CLineProfiler, ByCountProfilerMixin): """ def __call__(self, func): """ - Decorate a function, method, property, partial object etc. to - start the profiler on function entry and stop it on function - exit. + Decorate a function, method, :py:class:`property`, + :py:func:`~functools.partial` object etc. to start the profiler + on function entry and stop it on function exit. """ # The same object is returned when: # - `func` is a `types.FunctionType` which is already @@ -274,18 +426,35 @@ def wrap_callable(self, func): return func return super().wrap_callable(func) - def add_callable(self, func): + def add_callable(self, func, guard=None): """ - Register a function, method, property, partial object, etc. with - the underlying Cython profiler. + Register a function, method, :py:class:`property`, + :py:func:`~functools.partial` object, etc. with the underlying + Cython profiler. + + Args: + func (...): + Function, class/static/bound method, property, etc. + guard (Optional[Callable[[types.FunctionType], bool]]) + Optional checker callable, which takes a function object + and returns true(-y) if it *should not* be passed to + :py:meth:`.add_function()`. Defaults to checking + whether the function is already a profiling wrapper. Returns: 1 if any function is added to the profiler, 0 otherwise. + + Note: + This method should in general be called instead of the more + low-level :py:meth:`.add_function()`. """ + if guard is None: + guard = self._already_a_wrapper + nadded = 0 for impl in _get_underlying_functions(func): info, wrapped_by_this_prof = self._get_wrapper_info(impl) - if wrapped_by_this_prof: + if wrapped_by_this_prof if guard is None else guard(impl): continue if info: # It's still a profiling wrapper, just wrapped by @@ -315,30 +484,43 @@ def print_stats(self, stream=None, output_unit=None, stripzeros=False, def _add_namespace( self, namespace, *, - seen=None, scoping_policy=ScopingPolicy.NONE, wrap=False): + seen=None, + func_scoping_policy=ScopingPolicy.NONE, + class_scoping_policy=ScopingPolicy.NONE, + module_scoping_policy=ScopingPolicy.NONE, + wrap=False): + def func_guard(func): + return self._already_a_wrapper(func) or not func_check(func) + if seen is None: seen = set() count = 0 - add_cls = functools.partial(self._add_namespace, - seen=seen, - scoping_policy=scoping_policy, - wrap=wrap) + add_namespace = functools.partial( + self._add_namespace, + seen=seen, + func_scoping_policy=func_scoping_policy, + class_scoping_policy=class_scoping_policy, + module_scoping_policy=module_scoping_policy, + wrap=wrap) wrap_failures = {} - if isinstance(namespace, type): - check = scoping_policy._add_class_filter(namespace) - else: - check = scoping_policy._add_module_filter(namespace) + func_check = func_scoping_policy.get_filter(namespace, 'func') + cls_check = class_scoping_policy.get_filter(namespace, 'class') + mod_check = module_scoping_policy.get_filter(namespace, 'module') for attr, value in vars(namespace).items(): if id(value) in seen: continue seen.add(id(value)) if isinstance(value, type): - if check(value) and add_cls(value): + if cls_check(value) and add_namespace(value): + count += 1 + continue + elif isinstance(value, types.ModuleType): + if mod_check(value) and add_namespace(value): count += 1 continue try: - if not self.add_callable(value): + if not self.add_callable(value, guard=func_guard): continue except TypeError: # Not a callable (wrapper) continue @@ -371,12 +553,17 @@ def add_class( Args: cls (type): Class to be profiled. - scoping_policy (Union[str, ScopingPolicy]): - Whether (and how) to match the scope of member classes - and decide on whether to add them: + scoping_policy (Union[str, ScopingPolicy, ScopingPolicyDict]): + Whether (and how) to match the scope of members and + decide on whether to add them: see the documentation for :py:class:`ScopingPolicy`. Strings are converted to :py:class:`ScopingPolicy` instances in a case-insensitive manner. + Can also be a mapping from the keys ``'func'``, + ``'class'``, and ``'module'`` to + :py:class:`ScopingPolicy` objects or strings convertible + thereto, in which case different policies can be enacted + for these object types. wrap (bool): Whether to replace the wrapped members with wrappers which automatically enable/disable the profiler when @@ -386,9 +573,12 @@ def add_class( n (int): Number of members added to the profiler. """ - scoping_policy = ScopingPolicy(scoping_policy) + policies = ScopingPolicy.to_policies(scoping_policy) return self._add_namespace(cls, - scoping_policy=scoping_policy, wrap=wrap) + func_scoping_policy=policies['func'], + class_scoping_policy=policies['class'], + module_scoping_policy=policies['module'], + wrap=wrap) def add_module( self, mod, *, scoping_policy=ScopingPolicy.SIBLINGS, wrap=False): @@ -399,12 +589,17 @@ def add_module( Args: mod (ModuleType): Module to be profiled. - scoping_policy (Union[str, ScopingPolicy]): - Whether (and how) to match the scope of member classes - and decide on whether to add them; + scoping_policy (Union[str, ScopingPolicy, ScopingPolicyDict]): + Whether (and how) to match the scope of members and + decide on whether to add them: see the documentation for :py:class:`ScopingPolicy`. Strings are converted to :py:class:`ScopingPolicy` instances in a case-insensitive manner. + Can also be a mapping from the keys ``'func'``, + ``'class'``, and ``'module'`` to + :py:class:`ScopingPolicy` objects or strings convertible + thereto, in which case different policies can be enacted + for these object types. wrap (bool): Whether to replace the wrapped members with wrappers which automatically enable/disable the profiler when @@ -414,9 +609,12 @@ def add_module( n (int): Number of members added to the profiler. """ - scoping_policy = ScopingPolicy(scoping_policy) + policies = ScopingPolicy.to_policies(scoping_policy) return self._add_namespace(mod, - scoping_policy=scoping_policy, wrap=wrap) + func_scoping_policy=policies['func'], + class_scoping_policy=policies['class'], + module_scoping_policy=policies['module'], + wrap=wrap) def _get_wrapper_info(self, func): info = getattr(func, self._profiler_wrapped_marker, None) diff --git a/line_profiler/line_profiler.pyi b/line_profiler/line_profiler.pyi index 38865d95..44247bc9 100644 --- a/line_profiler/line_profiler.pyi +++ b/line_profiler/line_profiler.pyi @@ -6,7 +6,8 @@ from types import (FunctionType, MethodType, ModuleType, BuiltinFunctionType, BuiltinMethodType, ClassMethodDescriptorType, MethodDescriptorType, MethodWrapperType, WrapperDescriptorType) -from typing import overload, Any, Literal, Callable, List, Tuple, TypeVar +from typing import (overload, + Any, Callable, List, Literal, Tuple, TypeVar, TypedDict) try: from typing import ( # type: ignore[attr-defined] # noqa: F401 TypeIs) @@ -25,7 +26,6 @@ CLevelCallable = TypeVar('CLevelCallable', CallableLike = TypeVar('CallableLike', FunctionType, partial, property, cached_property, MethodType, staticmethod, classmethod, partialmethod) -ScopingPolicyOption = Literal['exact', 'descendants', 'siblings', 'none'] def is_c_level_callable(func: Any) -> TypeIs[CLevelCallable]: @@ -37,11 +37,49 @@ def load_ipython_extension(ip) -> None: class ScopingPolicy(StringEnum): - EXACT = auto() + CHILDREN = auto() DESCENDANTS = auto() SIBLINGS = auto() NONE = auto() + @overload + def get_filter( + self, + namespace: type | ModuleType, + obj_type: Literal['func']) -> Callable[[FunctionType], bool]: + ... + + @overload + def get_filter( + self, + namespace: type | ModuleType, + obj_type: Literal['class']) -> Callable[[type], bool]: + ... + + @overload + def get_filter( + self, + namespace: type | ModuleType, + obj_type: Literal['module']) -> Callable[[ModuleType], bool]: + ... + + @classmethod + def to_policies(cls, + policies: (str + | 'ScopingPolicy' + | ScopingPolicyDict)) -> _ScopingPolicyDict: + ... + + +ScopingPolicyDict = TypedDict('ScopingPolicyDict', + {'func': str | ScopingPolicy, + 'class': str | ScopingPolicy, + 'module': str | ScopingPolicy}) +_ScopingPolicyDict = TypedDict('_ScopingPolicyDict', + {'func': str | ScopingPolicy, + 'class': str | ScopingPolicy, + 'module': str | ScopingPolicy}) + class LineProfiler(CLineProfiler, ByCountProfilerMixin): @overload @@ -60,7 +98,9 @@ class LineProfiler(CLineProfiler, ByCountProfilerMixin): def __call__(self, func: Callable) -> FunctionType: ... - def add_callable(self, func) -> Literal[0, 1]: + def add_callable( + self, func, guard: (Callable[[FunctionType], bool] + | None) = None) -> Literal[0, 1]: ... def dump_stats(self, filename) -> None: @@ -78,13 +118,17 @@ class LineProfiler(CLineProfiler, ByCountProfilerMixin): def add_module( self, mod: ModuleType, *, - scoping_policy: ScopingPolicy | str = ScopingPolicy.SIBLINGS, + scoping_policy: (ScopingPolicy + | str + | ScopingPolicyDict) = ScopingPolicy.SIBLINGS, wrap: bool = False) -> int: ... def add_class( self, cls: type, *, - scoping_policy: ScopingPolicy | str = ScopingPolicy.SIBLINGS, + scoping_policy: (ScopingPolicy + | str + | ScopingPolicyDict) = ScopingPolicy.SIBLINGS, wrap: bool = False) -> int: ... diff --git a/tests/test_explicit_profile.py b/tests/test_explicit_profile.py index 761529f9..cd9addf9 100644 --- a/tests/test_explicit_profile.py +++ b/tests/test_explicit_profile.py @@ -606,7 +606,8 @@ def method(self): @pytest.mark.parametrize( ('scoping_policy', 'add_module_targets', 'add_class_targets'), - [('exact', + [('exact', {}, {'class3_method'}), + ('children', {'class2_method', 'child_class2_method'}, {'class3_method', 'child_class3_method'}), ('descendants', @@ -624,13 +625,13 @@ def method(self): 'class3_method', 'child_class3_method', 'other_class3_method'}, {'child_class1_method', 'class3_method', 'child_class3_method', 'other_class3_method'})]) -def test_profiler_scope_matching(monkeypatch, - scoping_policy, - add_module_targets, - add_class_targets): +def test_profiler_class_scope_matching(monkeypatch, + scoping_policy, + add_module_targets, + add_class_targets): """ - Test for the scope-matching strategies of the `LineProfiler.add_*()` - methods. + Test for the (class-)scope-matching strategies of the + `LineProfiler.add_*()` methods. """ def write(path, code=None): path.parent.mkdir(exist_ok=True, parents=True) @@ -655,7 +656,7 @@ def child_class1_method(self): write(pkg_dir / 'subpkg2' / '__init__.py', """ from ..submod1 import Class1 # Import from a sibling - from .submod3 import Class3 # Import from a descendant + from .submod3 import Class3 # Import descendant from a child class Class2: @@ -696,20 +697,25 @@ def other_class3_method(self): from my_pkg import subpkg2 from line_profiler import LineProfiler + policies = {'func': 'none', 'class': scoping_policy, + 'module': 'exact'} # Don't descend into submodules # Add a module profile = LineProfiler() - profile.add_module(subpkg2, scoping_policy=scoping_policy) + profile.add_module(subpkg2, scoping_policy=policies) assert len(profile.functions) == len(add_module_targets) added = {func.__name__ for func in profile.functions} assert added == set(add_module_targets) # Add a class profile = LineProfiler() - profile.add_class(subpkg2.Class3, scoping_policy=scoping_policy) + profile.add_class(subpkg2.Class3, scoping_policy=policies) assert len(profile.functions) == len(add_class_targets) added = {func.__name__ for func in profile.functions} assert added == set(add_class_targets) +# TODO: add similar tests for function and module scoping + + if __name__ == '__main__': ... test_simple_explicit_nonglobal_usage() From 18f8d0a7236b38ab7ec500d970147273c0c115c4 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 24 May 2025 07:15:45 +0200 Subject: [PATCH 29/70] Fixed test after rebasing on #345 --- tests/test_autoprofile.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/test_autoprofile.py b/tests/test_autoprofile.py index a0f2bf34..be87202b 100644 --- a/tests/test_autoprofile.py +++ b/tests/test_autoprofile.py @@ -1,4 +1,5 @@ import os +import re import subprocess import sys import shlex @@ -720,4 +721,12 @@ def descriptor(self): proc.check_returncode() for func in all_checked_funcs: - assert (f'Function: {func}' in raw_output) == (func in profiled_funcs) + if sys.version_info[:2] >= (3, 11) and func != 'function': + # Match qualnames, see PR #345 + prefix = r'.*\.' + else: + prefix = '' + in_output = re.search(f'^Function: {prefix}{func}', + raw_output, + re.MULTILINE) + assert bool(in_output) == (func in profiled_funcs) From 96dbed386bf1e7a305af1b3914b323380ab95aca Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 24 May 2025 09:10:59 +0200 Subject: [PATCH 30/70] Added scoping-policy tests line_profiler/line_profiler.py::ScopingPolicy Extended docstring tests/test_explicit_profile.py test_profiler_{module,func}_scope_matching() New tests in the same vein as `test_profiler_class_scope_matching()` for testing scoping --- line_profiler/line_profiler.py | 11 +- tests/test_explicit_profile.py | 214 +++++++++++++++++++++++++++++++-- 2 files changed, 212 insertions(+), 13 deletions(-) diff --git a/line_profiler/line_profiler.py b/line_profiler/line_profiler.py index f4feb18d..b3b6b961 100755 --- a/line_profiler/line_profiler.py +++ b/line_profiler/line_profiler.py @@ -145,6 +145,13 @@ class ScopingPolicy(StringEnum): * Their child classes, functions, and modules, ... * ... and so on + Note: + Since imported submodule module objects are by default + placed into the namespace of their parent-package module + objects, this functions largely identical to + :py:attr:`ScopingPolicy.CHILDREN` for descension from module + objects into other modules objects. + :py:attr:`ScopingPolicy.SIBLINGS` Only profile/descend into *sibling* and descendant objects, which are: @@ -164,7 +171,9 @@ class ScopingPolicy(StringEnum): namespaces recursively Note: - This is probably a very bad idea for module scoping; + This is probably a *very* bad idea for module scoping, + potentially resulting in accidentally recusing through a + significant portion of loaded modules; proceed with care. Note: diff --git a/tests/test_explicit_profile.py b/tests/test_explicit_profile.py index cd9addf9..acedc8f1 100644 --- a/tests/test_explicit_profile.py +++ b/tests/test_explicit_profile.py @@ -38,6 +38,26 @@ def __exit__(self, *_, **__): self.stack.close() +class restore_sys_modules: + """ + Restore :py:attr:`sys.modules` after exiting the context. + """ + def __enter__(self): + self.old = sys.modules.copy() + + def __exit__(self, *_, **__): + sys.modules.clear() + sys.modules.update(self.old) + + +def write(path, code=None): + path.parent.mkdir(exist_ok=True, parents=True) + if code is None: + path.touch() + else: + path.write_text(ub.codeblock(code)) + + def test_simple_explicit_nonglobal_usage(): """ python -c "from test_explicit_profile import *; test_simple_explicit_nonglobal_usage()" @@ -450,9 +470,6 @@ def test_profiler_add_methods(wrap_class, wrap_module, reset_enable_count): `line_profiler.autoprofile.autoprofile. _extend_line_profiler_for_profiling_imports()`) methods. """ - def write(path, code): - path.write_text(ub.codeblock(code)) - script = ub.codeblock(''' from line_profiler import LineProfiler from line_profiler.autoprofile.autoprofile import ( @@ -630,17 +647,13 @@ def test_profiler_class_scope_matching(monkeypatch, add_module_targets, add_class_targets): """ - Test for the (class-)scope-matching strategies of the + Test for the class-scope-matching strategies of the `LineProfiler.add_*()` methods. """ - def write(path, code=None): - path.parent.mkdir(exist_ok=True, parents=True) - if code is None: - path.touch() - else: - path.write_text(ub.codeblock(code)) + with ExitStack() as stack: + stack.enter_context(restore_sys_modules()) + curdir = stack.enter_context(enter_tmpdir()) - with enter_tmpdir() as curdir: pkg_dir = curdir / 'packages' / 'my_pkg' write(pkg_dir / '__init__.py') write(pkg_dir / 'submod1.py', @@ -713,7 +726,184 @@ def other_class3_method(self): assert added == set(add_class_targets) -# TODO: add similar tests for function and module scoping +@pytest.mark.parametrize( + ('scoping_policy', 'add_module_targets', 'add_subpackage_targets'), + [('exact', {'func4'}, {'class_method'}), + ('children', {'func4'}, {'class_method', 'func2'}), + ('descendants', {'func4'}, {'class_method', 'func2'}), + ('siblings', {'func4'}, {'class_method', 'func2', 'func3'}), + ('none', + {'func4', 'func5'}, + {'class_method', 'func2', 'func3', 'func4', 'func5'})]) +def test_profiler_module_scope_matching(monkeypatch, + scoping_policy, + add_module_targets, + add_subpackage_targets): + """ + Test for the module-scope-matching strategies of the + `LineProfiler.add_*()` methods. + """ + with ExitStack() as stack: + stack.enter_context(restore_sys_modules()) + curdir = stack.enter_context(enter_tmpdir()) + + pkg_dir = curdir / 'packages' / 'my_pkg' + write(pkg_dir / '__init__.py') + write(pkg_dir / 'subpkg1' / '__init__.py', + """ + import my_mod4 # Unrelated + from .. import submod3 # Sibling + from . import submod2 # Child + + + class Class: + @classmethod + def class_method(cls): + pass + + # We shouldn't descend into this no matter what + import my_mod5 as module + """) + write(pkg_dir / 'subpkg1' / 'submod2.py', + """ + def func2(): + pass + """) + write(pkg_dir / 'submod3.py', + """ + def func3(): + pass + """) + write(curdir / 'packages' / 'my_mod4.py', + """ + import my_mod5 # Unrelated + + + def func4(): + pass + """) + write(curdir / 'packages' / 'my_mod5.py', + """ + def func5(): + pass + """) + monkeypatch.syspath_prepend(pkg_dir.parent) + + import my_mod4 + from my_pkg import subpkg1 + from line_profiler import LineProfiler + + policies = {'func': 'none', 'class': 'children', + 'module': scoping_policy} + # Add a module + profile = LineProfiler() + profile.add_module(my_mod4, scoping_policy=policies) + assert len(profile.functions) == len(add_module_targets) + added = {func.__name__ for func in profile.functions} + assert added == set(add_module_targets) + # Add a subpackage + profile = LineProfiler() + profile.add_module(subpkg1, scoping_policy=policies) + assert len(profile.functions) == len(add_subpackage_targets) + added = {func.__name__ for func in profile.functions} + assert added == set(add_subpackage_targets) + # Add a class + profile = LineProfiler() + profile.add_class(subpkg1.Class, scoping_policy=policies) + assert [func.__name__ for func in profile.functions] == ['class_method'] + + +@pytest.mark.parametrize( + ('scoping_policy', 'add_module_targets', 'add_class_targets'), + [('exact', {'func1'}, {'method'}), + ('children', {'func1'}, {'method'}), + ('descendants', {'func1', 'func2'}, {'method', 'child_class_method'}), + ('siblings', + {'func1', 'func2', 'func3'}, + {'method', 'child_class_method', 'func1'}), + ('none', + {'func1', 'func2', 'func3', 'func4'}, + {'method', 'child_class_method', 'func1', 'another_func4'})]) +def test_profiler_func_scope_matching(monkeypatch, + scoping_policy, + add_module_targets, + add_class_targets): + """ + Test for the class-scope-matching strategies of the + `LineProfiler.add_*()` methods. + """ + with ExitStack() as stack: + stack.enter_context(restore_sys_modules()) + curdir = stack.enter_context(enter_tmpdir()) + + pkg_dir = curdir / 'packages' / 'my_pkg' + write(pkg_dir / '__init__.py') + write(pkg_dir / 'subpkg1' / '__init__.py', + """ + from ..submod3 import func3 # Sibling + from .submod2 import func2 # Descendant + from my_mod4 import func4 # Unrelated + + def func1(): + pass + + class Class: + def method(self): + pass + + class ChildClass: + @classmethod + def child_class_method(cls): + pass + + # Descendant + descdent_method = ChildClass.child_class_method + + # Sibling + sibling_method = staticmethod(func1) + + # Unrelated + from my_mod4 import another_func4 as imported_method + """) + write(pkg_dir / 'subpkg1' / 'submod2.py', + """ + def func2(): + pass + """) + write(pkg_dir / 'submod3.py', + """ + def func3(): + pass + """) + write(curdir / 'packages' / 'my_mod4.py', + """ + def func4(): + pass + + + def another_func4(_): + pass + """) + monkeypatch.syspath_prepend(pkg_dir.parent) + + from my_pkg import subpkg1 + from line_profiler import LineProfiler + + policies = {'func': scoping_policy, + # No descensions + 'class': 'exact', 'module': 'exact'} + # Add a module + profile = LineProfiler() + profile.add_module(subpkg1, scoping_policy=policies) + assert len(profile.functions) == len(add_module_targets) + added = {func.__name__ for func in profile.functions} + assert added == set(add_module_targets) + # Add a class + profile = LineProfiler() + profile.add_module(subpkg1.Class, scoping_policy=policies) + assert len(profile.functions) == len(add_class_targets) + added = {func.__name__ for func in profile.functions} + assert added == set(add_class_targets) if __name__ == '__main__': From 7d0f8a57f17da47d5230365415d0e244d581c1e9 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 24 May 2025 10:37:27 +0200 Subject: [PATCH 31/70] More sensible scoping defaults line_profiler/autoprofile/line_profiler_utils.py[i] add_imported_function_or_module(scoping_policy=...) - Updated default, documentation, and annotation - Now accepting `None` line_profiler/line_profiler.py[i] DEFAULT_SCOPING_POLICIES New defaults for `scoping_policy` ScopingPolicy.to_policies(scoping_policy=...) - Added default, updated documentation and annotation - Now accepting `None` (falls back to `DEFAULT_SCOPING_POLICIES`) LineProfiler.add_{class,module}(scoping_policy=...) - Updated default, documentation, and annotation - Now accepting `None` --- .../autoprofile/line_profiler_utils.py | 46 ++++++---- .../autoprofile/line_profiler_utils.pyi | 10 +- line_profiler/line_profiler.py | 91 +++++++++++++------ line_profiler/line_profiler.pyi | 18 ++-- 4 files changed, 103 insertions(+), 62 deletions(-) diff --git a/line_profiler/autoprofile/line_profiler_utils.py b/line_profiler/autoprofile/line_profiler_utils.py index 5f2984b7..c4e736d1 100644 --- a/line_profiler/autoprofile/line_profiler_utils.py +++ b/line_profiler/autoprofile/line_profiler_utils.py @@ -1,10 +1,8 @@ import inspect -from ..line_profiler import ScopingPolicy -def add_imported_function_or_module( - self, item, *, - scoping_policy=ScopingPolicy.SIBLINGS, wrap=False): +def add_imported_function_or_module(self, item, *, + scoping_policy=None, wrap=False): """ Method to add an object to :py:class:`~.line_profiler.LineProfiler` to be profiled. @@ -17,18 +15,27 @@ def add_imported_function_or_module( Args: item (Union[Callable, Type, ModuleType]): Object to be profiled. - scoping_policy (Union[ScopingPolicy, str]): - Whether (and how) to match the scope of member classes to - ``item`` (if a class or module) and decide on whether to add - them; - see the documentation for :py:class:`~.ScopingPolicy` for - details. - Strings are converted to :py:class:`~.ScopingPolicy` - instances in a case-insensitive manner. - Can also be a mapping from the keys ``'func'``, ``'class'``, - and ``'module'`` to :py:class:`~.ScopingPolicy` objects or - strings convertible thereto, in which case different - policies can be enacted for these object types. + scoping_policy (Union[ScopingPolicy, str, ScopingPolicyDict, \ +None]): + Whether (and how) to match the scope of members and decide + on whether to add them: + + :py:class:`str` (incl. :py:class:`~.ScopingPolicy`): + Strings are converted to :py:class:`~.ScopingPolicy` + instances in a case-insensitive manner, and the same + policy applies to all members. + + ``{'func': ..., 'class': ..., 'module': ...}`` + Mapping specifying individual policies to be enacted for + the corresponding member types. + + :py:const:`None` + The default, equivalent to + :py:data:`~line_profiler.line_profiler\ +.DEFAULT_SCOPING_POLICIES`. + + See :py:class:`line_profiler.line_profiler.ScopingPolicy` + and :py:meth:`~.ScopingPolicy.to_policies` for details. wrap (bool): Whether to replace the wrapped members with wrappers which automatically enable/disable the profiler when called. @@ -37,9 +44,14 @@ def add_imported_function_or_module( 1 if any function is added to the profiler, 0 otherwise. See also: + :py:data:`~line_profiler.line_profiler\ +.DEFAULT_SCOPING_POLICIES`, :py:meth:`.LineProfiler.add_callable()`, :py:meth:`.LineProfiler.add_module()`, - :py:meth:`.LineProfiler.add_class()` + :py:meth:`.LineProfiler.add_class()`, + :py:class:`~.ScopingPolicy`, + :py:meth:`ScopingPolicy.to_policies() \ +` """ if inspect.isclass(item): count = self.add_class(item, scoping_policy=scoping_policy, wrap=wrap) diff --git a/line_profiler/autoprofile/line_profiler_utils.pyi b/line_profiler/autoprofile/line_profiler_utils.pyi index 847737fa..a1116cbe 100644 --- a/line_profiler/autoprofile/line_profiler_utils.pyi +++ b/line_profiler/autoprofile/line_profiler_utils.pyi @@ -9,9 +9,8 @@ if TYPE_CHECKING: # Stub-only annotations @overload def add_imported_function_or_module( self, item: CLevelCallable | Any, - scoping_policy: (ScopingPolicy - | str - | ScopingPolicyDict) = ScopingPolicy.SIBLINGS, + scoping_policy: ( + ScopingPolicy | str | ScopingPolicyDict | None) = None, wrap: bool = False) -> Literal[0]: ... @@ -19,8 +18,7 @@ def add_imported_function_or_module( @overload def add_imported_function_or_module( self, item: CallableLike | type | ModuleType, - scoping_policy: (ScopingPolicy - | str - | ScopingPolicyDict) = ScopingPolicy.SIBLINGS, + scoping_policy: ( + ScopingPolicy | str | ScopingPolicyDict | None) = None, wrap: bool = False) -> Literal[0, 1]: ... diff --git a/line_profiler/line_profiler.py b/line_profiler/line_profiler.py index b3b6b961..0ac63887 100755 --- a/line_profiler/line_profiler.py +++ b/line_profiler/line_profiler.py @@ -43,6 +43,17 @@ types.MethodWrapperType, types.WrapperDescriptorType) +#: Default scoping policies: +#: +#: * Profile sibling and descendant functions +#: (:py:attr:`ScopingPolicy.SIBLINGS`) +#: * Descend ingo sibling and descendant classes +#: (:py:attr:`ScopingPolicy.SIBLINGS`) +#: * Don't descend into modules (:py:attr:`ScopingPolicy.EXACT`) +DEFAULT_SCOPING_POLICIES = types.MappingProxyType({'func': 'siblings', + 'class': 'siblings', + 'module': 'exact'}) + is_function = inspect.isfunction @@ -149,7 +160,7 @@ class ScopingPolicy(StringEnum): Since imported submodule module objects are by default placed into the namespace of their parent-package module objects, this functions largely identical to - :py:attr:`ScopingPolicy.CHILDREN` for descension from module + :py:attr:`ScopingPolicy.CHILDREN` for descent from module objects into other modules objects. :py:attr:`ScopingPolicy.SIBLINGS` @@ -172,7 +183,7 @@ class ScopingPolicy(StringEnum): Note: This is probably a *very* bad idea for module scoping, - potentially resulting in accidentally recusing through a + potentially resulting in accidentally recursing through a significant portion of loaded modules; proceed with care. @@ -250,19 +261,23 @@ def get_filter(self, namespace, obj_type): return method(namespace, is_class=(obj_type == 'class')) @classmethod - def to_policies(cls, policies): + def to_policies(cls, policies=None): """ Normalize ``policies`` into a dictionary of policies for various object types. Args: - policies (Union[str, ScopingPolicy, ScopingPolicyDict]): + policies (Union[str, ScopingPolicy, \ +ScopingPolicyDict, None]): :py:class:`ScopingPolicy`, string convertible thereto (case-insensitive), or a mapping containing such values - and the keys as outlined in the return value + and the keys as outlined in the return value; + the default :py:const:`None` is equivalent to + :py:data:`DEFAULT_SCOPING_POLICIES`. Returns: - normalized_policies (dict[str, ScopingPolicy]): + normalized_policies (dict[Literal['func', 'class', \ +'module'], ScopingPolicy]): Dictionary with the following key-value pairs: ``'func'`` @@ -300,6 +315,8 @@ def to_policies(cls, policies): ... KeyError: 'func' """ + if policies is None: + policies = DEFAULT_SCOPING_POLICIES if isinstance(policies, str): policy = cls(policies) return _ScopingPolicyDict( @@ -553,8 +570,7 @@ def func_guard(func): warnings.warn(msg, stacklevel=2) return count - def add_class( - self, cls, *, scoping_policy=ScopingPolicy.SIBLINGS, wrap=False): + def add_class(self, cls, *, scoping_policy=None, wrap=False): """ Add the members (callables (wrappers), methods, classes, ...) in a class' local namespace and profile them. @@ -562,17 +578,26 @@ def add_class( Args: cls (type): Class to be profiled. - scoping_policy (Union[str, ScopingPolicy, ScopingPolicyDict]): + scoping_policy (Union[str, ScopingPolicy, \ +ScopingPolicyDict, None]): Whether (and how) to match the scope of members and decide on whether to add them: - see the documentation for :py:class:`ScopingPolicy`. - Strings are converted to :py:class:`ScopingPolicy` - instances in a case-insensitive manner. - Can also be a mapping from the keys ``'func'``, - ``'class'``, and ``'module'`` to - :py:class:`ScopingPolicy` objects or strings convertible - thereto, in which case different policies can be enacted - for these object types. + + :py:class:`str` (incl. :py:class:`ScopingPolicy`): + Strings are converted to :py:class:`ScopingPolicy` + instances in a case-insensitive manner, and the same + policy applies to all members. + + ``{'func': ..., 'class': ..., 'module': ...}`` + Mapping specifying individual policies to be enacted + for the corresponding member types. + + :py:const:`None` + The default, equivalent to + :py:data:`DEFAULT_SCOPING_POLICIES`. + + See :py:class:`ScopingPolicy` and + :py:meth:`~ScopingPolicy.to_policies` for details. wrap (bool): Whether to replace the wrapped members with wrappers which automatically enable/disable the profiler when @@ -589,8 +614,7 @@ def add_class( module_scoping_policy=policies['module'], wrap=wrap) - def add_module( - self, mod, *, scoping_policy=ScopingPolicy.SIBLINGS, wrap=False): + def add_module(self, mod, *, scoping_policy=None, wrap=False): """ Add the members (callables (wrappers), methods, classes, ...) in a module's local namespace and profile them. @@ -598,17 +622,26 @@ def add_module( Args: mod (ModuleType): Module to be profiled. - scoping_policy (Union[str, ScopingPolicy, ScopingPolicyDict]): + scoping_policy (Union[str, ScopingPolicy, \ +ScopingPolicyDict, None]): Whether (and how) to match the scope of members and decide on whether to add them: - see the documentation for :py:class:`ScopingPolicy`. - Strings are converted to :py:class:`ScopingPolicy` - instances in a case-insensitive manner. - Can also be a mapping from the keys ``'func'``, - ``'class'``, and ``'module'`` to - :py:class:`ScopingPolicy` objects or strings convertible - thereto, in which case different policies can be enacted - for these object types. + + :py:class:`str` (incl. :py:class:`ScopingPolicy`): + Strings are converted to :py:class:`ScopingPolicy` + instances in a case-insensitive manner, and the same + policy applies to all members. + + ``{'func': ..., 'class': ..., 'module': ...}`` + Mapping specifying individual policies to be enacted + for the corresponding member types. + + :py:const:`None` + The default, equivalent to + :py:data:`DEFAULT_SCOPING_POLICIES`. + + See :py:class:`ScopingPolicy` and + :py:meth:`~ScopingPolicy.to_policies` for details. wrap (bool): Whether to replace the wrapped members with wrappers which automatically enable/disable the profiler when @@ -954,7 +987,7 @@ def load_stats(filename): def main(): """ - The line profiler CLI to view output from ``kernprof -l``. + The line profiler CLI to view output from :command:`kernprof -l`. """ def positive_float(value): val = float(value) diff --git a/line_profiler/line_profiler.pyi b/line_profiler/line_profiler.pyi index 44247bc9..7502bab8 100644 --- a/line_profiler/line_profiler.pyi +++ b/line_profiler/line_profiler.pyi @@ -64,10 +64,10 @@ class ScopingPolicy(StringEnum): ... @classmethod - def to_policies(cls, - policies: (str - | 'ScopingPolicy' - | ScopingPolicyDict)) -> _ScopingPolicyDict: + def to_policies( + cls, + policies: (str | 'ScopingPolicy' + | ScopingPolicyDict | None) = None) -> _ScopingPolicyDict: ... @@ -118,17 +118,15 @@ class LineProfiler(CLineProfiler, ByCountProfilerMixin): def add_module( self, mod: ModuleType, *, - scoping_policy: (ScopingPolicy - | str - | ScopingPolicyDict) = ScopingPolicy.SIBLINGS, + scoping_policy: ( + ScopingPolicy | str | ScopingPolicyDict | None) = None, wrap: bool = False) -> int: ... def add_class( self, cls: type, *, - scoping_policy: (ScopingPolicy - | str - | ScopingPolicyDict) = ScopingPolicy.SIBLINGS, + scoping_policy: ( + ScopingPolicy | str | ScopingPolicyDict | None) = None, wrap: bool = False) -> int: ... From 0bf3fab29743af567ce1aed493c794240f2d695f Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 26 May 2025 08:43:10 +0200 Subject: [PATCH 32/70] Doc fixes docs/source/auto/ line_profiler.autoprofile.rst Updated index line_profiler.autoprofile.ast_profile_transformer.rst Renamed from `*.ast_profle_transformer.rst` (see PR #325) line_profiler.autoprofile.{eager_preimports,run_module}.rst New boilerplate files for inclusing those modules --- ...line_profiler.autoprofile.ast_profile_transformer.rst} | 0 .../auto/line_profiler.autoprofile.eager_preimports.rst | 8 ++++++++ docs/source/auto/line_profiler.autoprofile.rst | 2 ++ docs/source/auto/line_profiler.autoprofile.run_module.rst | 8 ++++++++ 4 files changed, 18 insertions(+) rename docs/source/auto/{line_profiler.autoprofile.ast_profle_transformer.rst => line_profiler.autoprofile.ast_profile_transformer.rst} (100%) create mode 100644 docs/source/auto/line_profiler.autoprofile.eager_preimports.rst create mode 100644 docs/source/auto/line_profiler.autoprofile.run_module.rst diff --git a/docs/source/auto/line_profiler.autoprofile.ast_profle_transformer.rst b/docs/source/auto/line_profiler.autoprofile.ast_profile_transformer.rst similarity index 100% rename from docs/source/auto/line_profiler.autoprofile.ast_profle_transformer.rst rename to docs/source/auto/line_profiler.autoprofile.ast_profile_transformer.rst diff --git a/docs/source/auto/line_profiler.autoprofile.eager_preimports.rst b/docs/source/auto/line_profiler.autoprofile.eager_preimports.rst new file mode 100644 index 00000000..6803f2db --- /dev/null +++ b/docs/source/auto/line_profiler.autoprofile.eager_preimports.rst @@ -0,0 +1,8 @@ +line\_profiler.autoprofile.eager\_preimports module +=================================================== + +.. automodule:: line_profiler.autoprofile.eager_preimports + :members: + :undoc-members: + :show-inheritance: + :private-members: diff --git a/docs/source/auto/line_profiler.autoprofile.rst b/docs/source/auto/line_profiler.autoprofile.rst index 74ced100..4f6b40d7 100644 --- a/docs/source/auto/line_profiler.autoprofile.rst +++ b/docs/source/auto/line_profiler.autoprofile.rst @@ -10,8 +10,10 @@ Submodules line_profiler.autoprofile.ast_profile_transformer line_profiler.autoprofile.ast_tree_profiler line_profiler.autoprofile.autoprofile + line_profiler.autoprofile.eager_preimports line_profiler.autoprofile.line_profiler_utils line_profiler.autoprofile.profmod_extractor + line_profiler.autoprofile.run_module line_profiler.autoprofile.util_static Module contents diff --git a/docs/source/auto/line_profiler.autoprofile.run_module.rst b/docs/source/auto/line_profiler.autoprofile.run_module.rst new file mode 100644 index 00000000..238d24e2 --- /dev/null +++ b/docs/source/auto/line_profiler.autoprofile.run_module.rst @@ -0,0 +1,8 @@ +line\_profiler.autoprofile.run\_module module +============================================= + +.. automodule:: line_profiler.autoprofile.run_module + :members: + :undoc-members: + :show-inheritance: + :private-members: From 3b0e94c11f475a62024bafdf25b8abf93fad8cb1 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 26 May 2025 10:32:50 +0200 Subject: [PATCH 33/70] WIP: recurively profile packages kernprof.py main() Added flag `--recursive-prof-mod` for controlling recursive eager pre-imports into packages _write_preimports() Added argument `recursive_prof_mod` to handle the above _main() Added handling for the above line_profiler/autoprofile/eager_preimports.py[i] Updated docstrings to be more `sphinx`-friendly write_eager_import_module() - Added argument `recurse` for controlling the generation of recursive eager imports - Added argument `indent` for controlling indentation of the written module - Added `try: ... except ImportError: ...` blocks around imports TODO: - Update `kernprof`'s docstring - Add help text for `--recursive-prof-mod` - Add tests for `--recursive-prof-mod` and `write_eager_import_module(recurse=...)` --- kernprof.py | 93 ++++++--- line_profiler/autoprofile/eager_preimports.py | 182 +++++++++++++----- .../autoprofile/eager_preimports.pyi | 4 +- 3 files changed, 196 insertions(+), 83 deletions(-) diff --git a/kernprof.py b/kernprof.py index 118fdd46..68ed4a14 100755 --- a/kernprof.py +++ b/kernprof.py @@ -482,6 +482,13 @@ def positive_float(value): "Adding the current script/module profiles the " "entirety of it. " "Only works with line_profiler -l, --line-by-line.") + parser.add_argument('--recursive-prof-mod', + action='append', + const=True, + metavar=("{path/to/script | object.dotted.path}" + "[,...]"), + nargs='?', + help="...") # TODO parser.add_argument('--no-preimports', action='store_true', help="Instead of eagerly importing all profiling " @@ -561,7 +568,7 @@ def _write_tempfile(source, content, options, tmpdir): suffix='.' + extension) -def _write_preimports(prof, prof_mod, exclude): +def _write_preimports(prof, prof_mod, recursive_prof_mod, exclude): """ Called by :py:func:`main()` to handle eager pre-imports; not to be invoked on its own. @@ -572,35 +579,41 @@ def _write_preimports(prof, prof_mod, exclude): from line_profiler.autoprofile.autoprofile import ( _extend_line_profiler_for_profiling_imports as upgrade_profiler) - filtered_targets = [] - invalid_targets = [] - for target in prof_mod: - if is_dotted_path(target): - filtered_targets.append(target) - continue - # Paths already normalized by `_normalize_profiling_targets()` - if not os.path.exists(target): - invalid_targets.append(target) - continue - if any(os.path.samefile(target, excluded) for excluded in exclude): - # Ignore the script to be run in eager importing - # (`line_profiler.autoprofile.autoprofile.run()` will handle - # it) - continue - modname = modpath_to_modname(target) - if modname is None: # Not import-able - invalid_targets.append(target) - continue - filtered_targets.append(modname) + filtered_prof_mod = [] + filtered_rpm = [] + to_filter = [(filtered_prof_mod, prof_mod)] + if recursive_prof_mod in (True, False): + filtered_rpm = recursive_prof_mod + else: + to_filter.append((filtered_rpm, recursive_prof_mod)) + invalid_targets = set() + for filtered, targets in to_filter: + for target in targets: + if is_dotted_path(target): + filtered.append(target) + continue + # Paths already normalized by `_normalize_profiling_targets()` + if not os.path.exists(target): + invalid_targets.add(target) + continue + if any(os.path.samefile(target, excluded) for excluded in exclude): + # Ignore the script to be run in eager importing + # (`line_profiler.autoprofile.autoprofile.run()` will handle + # it) + continue + modname = modpath_to_modname(target) + if modname is None: # Not import-able + invalid_targets.add(target) + continue + filtered.append(modname) if invalid_targets: - invalid_targets = sorted(set(invalid_targets)) msg = ('{} profile-on-import target{} cannot be converted to ' 'dotted-path form: {!r}' .format(len(invalid_targets), '' if len(invalid_targets) == 1 else 's', - invalid_targets)) + sorted(invalid_targets))) warnings.warn(msg) - if not filtered_targets: + if not filtered_prof_mod: return # - We could've done everything in-memory with `io.StringIO` and # `exec()`, but that results in indecipherable tracebacks should @@ -618,7 +631,9 @@ def _write_preimports(prof, prof_mod, exclude): with tempfile.TemporaryDirectory() as tmpdir: temp_mod_path = os.path.join(tmpdir, temp_mod_name + '.py') with open(temp_mod_path, mode='w') as fobj: - write_eager_import_module(filtered_targets, stream=fobj) + write_eager_import_module(filtered_prof_mod, + recurse=filtered_rpm, + stream=fobj) ns = {} # Use a fresh namespace try: execfile(temp_mod_path, ns, ns) @@ -698,12 +713,29 @@ def _main(options, module=False): # If using eager pre-imports, write a dummy module which contains # all those imports and marks them for profiling, then run it + all_prof_mod_targets = [] if options.prof_mod: # Note: `prof_mod` entries can be filenames (which can contain # commas), so check against existing filenames before splitting # them options.prof_mod = _normalize_profiling_targets(options.prof_mod) - if not options.prof_mod: + all_prof_mod_targets.extend(options.prof_mod) + if options.recursive_prof_mod: + rpm_has_true = any(target in (True,) + for target in options.recursive_prof_mod) + rpm_targets = _normalize_profiling_targets( + target for target in options.recursive_prof_mod + if target not in (True,)) + if rpm_has_true and not rpm_targets: + options.recursive_prof_mod = True + elif rpm_targets: + options.recursive_prof_mod = rpm_targets + all_prof_mod_targets.extend(rpm_targets) + else: + options.recursive_prof_mod = False + else: + options.recursive_prof_mod = False + if not all_prof_mod_targets: options.no_preimports = True if options.line_by_line and not options.no_preimports: # We assume most items in `.prof_mod` to be import-able without @@ -712,7 +744,10 @@ def _main(options, module=False): # even have a `if __name__ == '__main__': ...` guard. So don't # eager-import it. exclude = set() if module else {script_file} - _write_preimports(prof, options.prof_mod, exclude) + _write_preimports(prof, + options.prof_mod, + options.recursive_prof_mod, + exclude) if options.output_interval: rt = RepeatedTimer(max(options.output_interval, 1), prof.dump_stats, options.outfile) @@ -725,10 +760,10 @@ def _main(options, module=False): rmod_ = functools.partial(run_module, run_name='__main__', alter_sys=True) ns = locals() - if options.prof_mod and options.line_by_line: + if all_prof_mod_targets and options.line_by_line: from line_profiler.autoprofile import autoprofile autoprofile.run(script_file, ns, - prof_mod=options.prof_mod, + prof_mod=all_prof_mod_targets, profile_imports=options.prof_imports, as_module=module is not None) elif module and options.builtin: diff --git a/line_profiler/autoprofile/eager_preimports.py b/line_profiler/autoprofile/eager_preimports.py index 5459fbf1..a10088ce 100644 --- a/line_profiler/autoprofile/eager_preimports.py +++ b/line_profiler/autoprofile/eager_preimports.py @@ -1,14 +1,18 @@ """ Tools for eagerly pre-importing everything as specified in -`line_profiler.autoprof.run(prof_mod=...)`. +``line_profiler.autoprof.run(prof_mod=...)``. """ import ast import functools import itertools +from collections.abc import Collection from keyword import iskeyword from importlib.util import find_spec -from textwrap import dedent +from os.path import isdir +from pkgutil import walk_packages +from textwrap import dedent, indent as indent_ from warnings import warn +from .util_static import modname_to_modpath def is_dotted_path(obj): @@ -51,21 +55,26 @@ def split_dotted_path(dotted_path): Arguments: dotted_path (str): Dotted path indicating an import target (module, package, or - a `from ... import ...`-able name under that), or an object - accessible via (chained) attribute access thereon + a ``from ... import ...``-able name under that), or an + object accessible via (chained) attribute access thereon Returns: module, target (tuple[str, Union[str, None]]): - - module: dotted path indicating the module that should be + + * ``module``: dotted path indicating the module that should be imported - - target: dotted path indicating the chained attribute access - target on the imported module corresponding to `dotted_path`; - if the import is just a module, this is set to `None` + * ``target``: dotted path indicating the chained-attribute + access target on the imported module corresponding to + ``dotted_path``; + if the import is just a module, this is set to + :py:const:`None` Raises: - - `TypeError` if `dotted_path` is not a dotted path (Python - identifiers joined by periods) - - `ModuleNotFoundError` if a matching module cannot be found + TypeError + If ``dotted_path`` is not a dotted path (Python identifiers + joined by periods) + ModuleNotFoundError + If a matching module cannot be found Example: >>> split_dotted_path('importlib.util.find_spec') @@ -117,10 +126,10 @@ def strip(s): class LoadedNameFinder(ast.NodeVisitor): """ Find the names loaded in an AST. A name is considered to be loaded - if it appears with the context `ast.Load()` and is not an argument - of any surrounding function-definition contexts - (`def func(...): ...`, `async def func(...): ...`, or - `lambda ...: ...`). + if it appears with the context :py:class:`ast.Load()` and is not an + argument of any surrounding function-definition contexts + (``def func(...): ...``, ``async def func(...): ...``, or + ``lambda ...: ...``). Example: >>> import ast @@ -213,7 +222,9 @@ def propose_names(prefixes): def write_eager_import_module(dotted_paths, stream=None, *, - adder='profile.add_imported_function_or_module'): + recurse=False, + adder='profile.add_imported_function_or_module', + indent=' '): r""" Write a module which autoprofiles all its imports. @@ -224,23 +235,41 @@ def write_eager_import_module(dotted_paths, stream=None, *, stream (Union[TextIO, None]): Optional text-mode writable file object to which to write the module + recurse (Union[Collection[str], bool]): + Dotted paths (strings of period-joined identifiers) + indicating the profiling targets that should be recursed + into if they are packages; + can also be a boolean value, indicating: + + :py:const:`True` + Recurse into any entry in ``dotted_paths`` that is a + package + :py:const:`False` + Don't recurse into any entry adder (str): - Single-line string `ast.parse(mode='eval')`-able to a single - expression, indicating the callable (which is assumed to - exist in the builtin namespace by the time the module is + Single-line string ``ast.parse(mode='eval')``-able to a + single expression, indicating the callable (which is assumed + to exist in the builtin namespace by the time the module is executed) to be called to add the profiling target + indent (str): + Single-line, non-empty whitespace string to indent the + output with Side effects: - - `stream` (or stdout if none) written to - - Warning issued if the module can't be located for one or more + * ``stream`` (or :py:data:`sys.stdout` if :py:const:`None`) + written to + * Warning issued if the module can't be located for one or more dotted paths Raises: - - `TypeError` if `adder` is not a string - - `ValueError` if `adder` is a non-single-line string or is not - parsable to a single expression - - `TypeError` if `dotted_paths` is not a collection of dotted - paths + TypeError + * If ``adder`` and ``indent`` are not strings + * If ``dotted_paths`` is not a collection of dotted paths + ValueError + * If ``adder`` is a non-single-line string or is not + parsable to a single expression + * If ``indent`` isn't single-line, non-empty, and + whitespace Example: >>> import io @@ -266,20 +295,27 @@ def write_eager_import_module(dotted_paths, stream=None, *, ... add = profile.add_imported_function_or_module ... failures = [] ... - ... import importlib.abc as module - ... ... try: - ... add(module.Loader.exec_module) - ... except AttributeError: - ... failures.append('importlib.abc.Loader.exec_module') - ... try: - ... add(module.Loader.find_module) - ... except AttributeError: - ... failures.append('importlib.abc.Loader.find_module') + ... import importlib.abc as module + ... except ImportError: + ... pass + ... else: + ... try: + ... add(module.Loader.exec_module) + ... except AttributeError: + ... failures.append('importlib.abc.Loader.exec_module') + ... try: + ... add(module.Loader.find_module) + ... except AttributeError: + ... failures.append('importlib.abc.Loader.find_module') ... - ... import importlib.util as module + ... try: + ... import importlib.util as module + ... except ImportError: + ... pass + ... else: + ... add(module) ... - ... add(module) ... ... if failures: ... import warnings @@ -309,6 +345,15 @@ def write_eager_import_module(dotted_paths, stream=None, *, raise AdderError(f'adder = {adder!r}: ' 'expected a single-line string parsable to a single ' 'expression') + if not isinstance(indent, str): + IndentError = TypeError + elif len(indent.splitlines()) == 1 and indent.isspace(): + IndentError = None + else: + IndentError = ValueError + if IndentError: + raise IndentError(f'indent = {indent!r}: ' + 'expected a single-line non-empty whitespace string') # Get the names loaded by `adder`; # these names are not allowed in the namespace @@ -331,6 +376,13 @@ def write_eager_import_module(dotted_paths, stream=None, *, if name not in forbidden_names) # Figure out the import targets to profile + dotted_paths = set(dotted_paths) + if isinstance(recurse, Collection): + recurse = set(recurse) + else: + recurse = dotted_paths if recurse else set() + dotted_paths |= recurse + imports = {} unknown_locs = [] for path in sorted(set(dotted_paths)): @@ -339,7 +391,17 @@ def write_eager_import_module(dotted_paths, stream=None, *, except ModuleNotFoundError: unknown_locs.append(path) continue - imports.setdefault(module, []).append(target) + if path in recurse and target is None: + recurse_root = modname_to_modpath(path, hide_init=True) + if recurse_root and not isdir(recurse_root): + recurse_root = None + else: # Not a recurse target nor a module + recurse_root = None + imports.setdefault(module, set()).add(target) + # FIXME: how do we handle namespace packages? + if recurse_root is not None: + for info in walk_packages([recurse_root], prefix=module + '.'): + imports.setdefault(info.name, set()).add(None) # Warn against failed imports if unknown_locs: @@ -353,28 +415,42 @@ def write_eager_import_module(dotted_paths, stream=None, *, write = functools.partial(print, file=stream) write(f'{adder_name} = {adder}\n{failures_name} = []') for module, targets in imports.items(): - write(f'\nimport {module} as {module_name}\n') - for target in targets: - if target is None: - write(f'{adder_name}({module_name})') - continue + assert targets + write('\n' + + strip(f""" + try: + {indent}import {module} as {module_name} + except ImportError: + {indent}pass + else: + """)) + chunks = [] + try: + targets.remove(None) + except KeyError: # Not found + pass + else: # Add the whole module + chunks.append(f'{adder_name}({module_name})') + for target in sorted(targets): path = f'{module}.{target}' - write(strip(f""" + chunks.append(strip(f""" try: - {adder_name}({module_name}.{target}) + {indent}{adder_name}({module_name}.{target}) except AttributeError: - {failures_name}.append({path!r}) + {indent}{failures_name}.append({path!r}) """)) + for chunk in chunks: + write(indent_(chunk, indent)) # Issue a warning if any of the targets doesn't exist if imports: - write('') + write('\n') write(strip(f""" if {failures_name}: - import warnings + {indent}import warnings - msg = '{{}} target{{}} cannot be imported: {{!r}}'.format( - len({failures_name}), - '' if len({failures_name}) == 1 else 's', - {failures_name}) - warnings.warn(msg, stacklevel=2) + {indent}msg = '{{}} target{{}} cannot be imported: {{!r}}'.format( + {indent * 2}len({failures_name}), + {indent * 2}'' if len({failures_name}) == 1 else 's', + {indent * 2}{failures_name}) + {indent}warnings.warn(msg, stacklevel=2) """)) diff --git a/line_profiler/autoprofile/eager_preimports.pyi b/line_profiler/autoprofile/eager_preimports.pyi index 5f312f18..c8b7b29a 100644 --- a/line_profiler/autoprofile/eager_preimports.pyi +++ b/line_profiler/autoprofile/eager_preimports.pyi @@ -44,5 +44,7 @@ def propose_names(prefixes: Collection[str]) -> Generator[str, None, None]: def write_eager_import_module( dotted_paths: Collection[str], stream: Union[TextIO, None] = None, *, - adder: str = 'profile.add_imported_function_or_module') -> None: + recurse: Union[Collection[str], bool] = False, + adder: str = 'profile.add_imported_function_or_module', + indent: str = ' ') -> None: ... From c9a961622e092ea6b0eb202d26727fa99766f7c9 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 26 May 2025 10:54:39 +0200 Subject: [PATCH 34/70] Doc updates CHANGELOG.rst Updated entry kernprof.py - Updated docstring - Added help text to the `--recursive-prof-mod` flag --- CHANGELOG.rst | 2 +- kernprof.py | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6b96a08e..ea79d10d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -15,7 +15,7 @@ Changes * ENH: In Python >=3.11, profiled objects are reported using their qualified name. * ENH: Highlight final summary using rich if enabled * ENH: Made it possible to use multiple profiler instances simultaneously -* ENH: ``kernprof --prof-mod`` target entities are now imported and profiled regardless of whether they are directly imported in the run script/module/code (old behavior recoed by passing ``--no-preimports``); made on-import profiling more aggressive so that it doesn't miss entities like class methods and properties +* ENH: ``kernprof --prof-mod`` target entities are now imported and profiled regardless of whether they are directly imported in the run script/module/code (old behavior restored by passing ``--no-preimports``; recurse into packages with ``--recursive-prof-mod``); made on-import profiling more aggressive so that it doesn't miss entities like class methods and properties 4.2.0 ~~~~~ diff --git a/kernprof.py b/kernprof.py index 68ed4a14..1a8e5d58 100755 --- a/kernprof.py +++ b/kernprof.py @@ -73,7 +73,8 @@ def main(): .. code:: - usage: kernprof [-h] [-V] [-l] [-b] [-o OUTFILE] [-s SETUP] [-v] [-r] [-u UNIT] [-z] [-i [OUTPUT_INTERVAL]] [-p PROF_MOD] [--prof-imports] + usage: kernprof [-h] [-V] [-l] [-b] [-o OUTFILE] [-s SETUP] [-v] [-r] [-u UNIT] [-z] [-i [OUTPUT_INTERVAL]] [-p {path/to/script | object.dotted.path}[,...]] + [--recursive-prof-mod [{path/to/script | object.dotted.path}[,...]]] [--no-preimports] [--prof-imports] {path/to/script | -m path.to.module | -c "literal code"} ... Run and profile a python script. @@ -101,6 +102,10 @@ def main(): -p, --prof-mod {path/to/script | object.dotted.path}[,...] List of modules, functions and/or classes to profile specified by their name or path. These profiling targets can be supplied both as comma-separated items, or separately with multiple copies of this flag. Adding the current script/module profiles the entirety of it. Only works with line_profiler -l, + --line-by-line. + --recursive-prof-mod [{path/to/script | object.dotted.path}[,...]] + List of packages to recurse into, profiling each of the submodules. The semantics are the same as --prof-mod. Only works with line_profiler -l, --line- + by-line. --no-preimports Instead of eagerly importing all profiling targets specified via -p and profiling them, only profile those that are directly imported in the profiled code. Only works with line_profiler -l, --line-by-line. --prof-imports If specified, modules specified to `--prof-mod` will also autoprofile modules that they import. Only works with line_profiler -l, --line-by-line @@ -488,7 +493,10 @@ def positive_float(value): metavar=("{path/to/script | object.dotted.path}" "[,...]"), nargs='?', - help="...") # TODO + help="List of packages to recurse into, profiling " + "each of the submodules. The semantics are the " + "same as --prof-mod. " + "Only works with line_profiler -l, --line-by-line.") parser.add_argument('--no-preimports', action='store_true', help="Instead of eagerly importing all profiling " From 6a57b726010651064ff9867d33f6c23e7d4ea927 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 26 May 2025 17:19:37 +0200 Subject: [PATCH 35/70] Simplified `kernprof` kernprof.py __doc__ Updated docstring main() Rolled back last two commits; no logner supports the `--recursive-prof-mod` flag _write_preimports() No longer takes the `recursive_prof_mod` argument, instead recursing into packages by default unless the package name is suffixed with `.__init__`, which limits profiling to the local scope thereof tests/test_autoprofile.py _write_demo_module() Added an import on the subpackage so that it has something in its local namespace test_autoprofile_exec_module() Added subtests which tests recursion into packages --- kernprof.py | 99 ++++++++++++++------------------------- tests/test_autoprofile.py | 14 +++++- 2 files changed, 46 insertions(+), 67 deletions(-) diff --git a/kernprof.py b/kernprof.py index 1a8e5d58..647779eb 100755 --- a/kernprof.py +++ b/kernprof.py @@ -73,8 +73,7 @@ def main(): .. code:: - usage: kernprof [-h] [-V] [-l] [-b] [-o OUTFILE] [-s SETUP] [-v] [-r] [-u UNIT] [-z] [-i [OUTPUT_INTERVAL]] [-p {path/to/script | object.dotted.path}[,...]] - [--recursive-prof-mod [{path/to/script | object.dotted.path}[,...]]] [--no-preimports] [--prof-imports] + usage: kernprof [-h] [-V] [-l] [-b] [-o OUTFILE] [-s SETUP] [-v] [-r] [-u UNIT] [-z] [-i [OUTPUT_INTERVAL]] [-p PROF_MOD] [--prof-imports] {path/to/script | -m path.to.module | -c "literal code"} ... Run and profile a python script. @@ -101,11 +100,8 @@ def main(): disabled. -p, --prof-mod {path/to/script | object.dotted.path}[,...] List of modules, functions and/or classes to profile specified by their name or path. These profiling targets can be supplied both as comma-separated - items, or separately with multiple copies of this flag. Adding the current script/module profiles the entirety of it. Only works with line_profiler -l, - --line-by-line. - --recursive-prof-mod [{path/to/script | object.dotted.path}[,...]] - List of packages to recurse into, profiling each of the submodules. The semantics are the same as --prof-mod. Only works with line_profiler -l, --line- - by-line. + items, or separately with multiple copies of this flag. Packages are automatically recursed into unless they are specified with `.__init__`. Adding + the current script/module profiles the entirety of it. Only works with line_profiler -l, --line-by-line. --no-preimports Instead of eagerly importing all profiling targets specified via -p and profiling them, only profile those that are directly imported in the profiled code. Only works with line_profiler -l, --line-by-line. --prof-imports If specified, modules specified to `--prof-mod` will also autoprofile modules that they import. Only works with line_profiler -l, --line-by-line @@ -128,6 +124,7 @@ def main(): import threading import asyncio # NOQA import concurrent.futures # NOQA +import pkgutil import tempfile import time import traceback @@ -484,19 +481,11 @@ def positive_float(value): "These profiling targets can be supplied both as " "comma-separated items, or separately with " "multiple copies of this flag. " + "Packages are automatically recursed into unless " + "they are specified with `.__init__`. " "Adding the current script/module profiles the " "entirety of it. " "Only works with line_profiler -l, --line-by-line.") - parser.add_argument('--recursive-prof-mod', - action='append', - const=True, - metavar=("{path/to/script | object.dotted.path}" - "[,...]"), - nargs='?', - help="List of packages to recurse into, profiling " - "each of the submodules. The semantics are the " - "same as --prof-mod. " - "Only works with line_profiler -l, --line-by-line.") parser.add_argument('--no-preimports', action='store_true', help="Instead of eagerly importing all profiling " @@ -576,7 +565,7 @@ def _write_tempfile(source, content, options, tmpdir): suffix='.' + extension) -def _write_preimports(prof, prof_mod, recursive_prof_mod, exclude): +def _write_preimports(prof, prof_mod, exclude): """ Called by :py:func:`main()` to handle eager pre-imports; not to be invoked on its own. @@ -587,41 +576,41 @@ def _write_preimports(prof, prof_mod, recursive_prof_mod, exclude): from line_profiler.autoprofile.autoprofile import ( _extend_line_profiler_for_profiling_imports as upgrade_profiler) - filtered_prof_mod = [] - filtered_rpm = [] - to_filter = [(filtered_prof_mod, prof_mod)] - if recursive_prof_mod in (True, False): - filtered_rpm = recursive_prof_mod - else: - to_filter.append((filtered_rpm, recursive_prof_mod)) - invalid_targets = set() - for filtered, targets in to_filter: - for target in targets: - if is_dotted_path(target): - filtered.append(target) - continue + filtered_targets = [] + recurse_targets = [] + invalid_targets = [] + for target in prof_mod: + if is_dotted_path(target): + modname = target + else: # Paths already normalized by `_normalize_profiling_targets()` if not os.path.exists(target): - invalid_targets.add(target) + invalid_targets.append(target) continue if any(os.path.samefile(target, excluded) for excluded in exclude): # Ignore the script to be run in eager importing # (`line_profiler.autoprofile.autoprofile.run()` will handle # it) continue - modname = modpath_to_modname(target) - if modname is None: # Not import-able - invalid_targets.add(target) - continue - filtered.append(modname) + modname = modpath_to_modname(target, hide_init=False) + if modname is None: # Not import-able + invalid_targets.append(target) + continue + if modname.endswith('.__init__'): + modname = modname.rpartition('.')[0] + targets = filtered_targets + else: + targets = recurse_targets + targets.append(modname) if invalid_targets: + invalid_targets = sorted(set(invalid_targets)) msg = ('{} profile-on-import target{} cannot be converted to ' 'dotted-path form: {!r}' .format(len(invalid_targets), '' if len(invalid_targets) == 1 else 's', - sorted(invalid_targets))) + invalid_targets)) warnings.warn(msg) - if not filtered_prof_mod: + if not (filtered_targets or recurse_targets): return # - We could've done everything in-memory with `io.StringIO` and # `exec()`, but that results in indecipherable tracebacks should @@ -639,8 +628,8 @@ def _write_preimports(prof, prof_mod, recursive_prof_mod, exclude): with tempfile.TemporaryDirectory() as tmpdir: temp_mod_path = os.path.join(tmpdir, temp_mod_name + '.py') with open(temp_mod_path, mode='w') as fobj: - write_eager_import_module(filtered_prof_mod, - recurse=filtered_rpm, + write_eager_import_module(filtered_targets, + recurse=recurse_targets, stream=fobj) ns = {} # Use a fresh namespace try: @@ -721,29 +710,12 @@ def _main(options, module=False): # If using eager pre-imports, write a dummy module which contains # all those imports and marks them for profiling, then run it - all_prof_mod_targets = [] if options.prof_mod: # Note: `prof_mod` entries can be filenames (which can contain # commas), so check against existing filenames before splitting # them options.prof_mod = _normalize_profiling_targets(options.prof_mod) - all_prof_mod_targets.extend(options.prof_mod) - if options.recursive_prof_mod: - rpm_has_true = any(target in (True,) - for target in options.recursive_prof_mod) - rpm_targets = _normalize_profiling_targets( - target for target in options.recursive_prof_mod - if target not in (True,)) - if rpm_has_true and not rpm_targets: - options.recursive_prof_mod = True - elif rpm_targets: - options.recursive_prof_mod = rpm_targets - all_prof_mod_targets.extend(rpm_targets) - else: - options.recursive_prof_mod = False - else: - options.recursive_prof_mod = False - if not all_prof_mod_targets: + if not options.prof_mod: options.no_preimports = True if options.line_by_line and not options.no_preimports: # We assume most items in `.prof_mod` to be import-able without @@ -752,10 +724,7 @@ def _main(options, module=False): # even have a `if __name__ == '__main__': ...` guard. So don't # eager-import it. exclude = set() if module else {script_file} - _write_preimports(prof, - options.prof_mod, - options.recursive_prof_mod, - exclude) + _write_preimports(prof, options.prof_mod, exclude) if options.output_interval: rt = RepeatedTimer(max(options.output_interval, 1), prof.dump_stats, options.outfile) @@ -768,10 +737,10 @@ def _main(options, module=False): rmod_ = functools.partial(run_module, run_name='__main__', alter_sys=True) ns = locals() - if all_prof_mod_targets and options.line_by_line: + if options.prof_mod and options.line_by_line: from line_profiler.autoprofile import autoprofile autoprofile.run(script_file, ns, - prof_mod=all_prof_mod_targets, + prof_mod=options.prof_mod, profile_imports=options.prof_imports, as_module=module is not None) elif module and options.builtin: diff --git a/tests/test_autoprofile.py b/tests/test_autoprofile.py index be87202b..7f82f85b 100644 --- a/tests/test_autoprofile.py +++ b/tests/test_autoprofile.py @@ -200,7 +200,10 @@ def _write_demo_module(temp_dpath): (temp_dpath / 'test_mod/subpkg').ensuredir() (temp_dpath / 'test_mod/__init__.py').touch() - (temp_dpath / 'test_mod/subpkg/__init__.py').touch() + (temp_dpath / 'test_mod/subpkg/__init__.py').write_text(ub.codeblock( + ''' + from .submod3 import add_three + ''')) (temp_dpath / 'test_mod/__main__.py').write_text(ub.codeblock( ''' @@ -503,7 +506,14 @@ def test_autoprofile_exec_package(use_kernprof_exec, prof_mod, '--prof-imports', {'add_one', 'add_two', 'add_four', 'add_operator', '_main'}), (False, None, '--prof-imports', {}), - (True, None, '--prof-imports', {})]) + (True, None, '--prof-imports', {}), + # Packages are descended into by default, unless they are specified + # with `.__init__` + (False, 'test_mod', '', + {'add_one', 'add_two', 'add_three', 'add_four', 'add_operator', + '_main'}), + (False, 'test_mod.subpkg', '', {'add_three', 'add_four', '_main'}), + (False, 'test_mod.subpkg.__init__', '', {'add_three'})]) def test_autoprofile_exec_module(use_kernprof_exec, prof_mod, flags, profiled_funcs): """ From 8339e06297d85abaebe06bf164ee97b97111eb80 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 26 May 2025 17:52:51 +0200 Subject: [PATCH 36/70] Cleanup in `kernprof` kernprof.py - Removed unused import `pkgutil` - Updated Note in docstring to document descent into packages --- kernprof.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/kernprof.py b/kernprof.py index 647779eb..29088eb5 100755 --- a/kernprof.py +++ b/kernprof.py @@ -110,10 +110,17 @@ def main(): New in 4.3.0: For more intuitive profiling behavior, profiling targets in :option:`!--prof-mod` (except the profiled script/code) - are now eagerly pre-imported to be profiled - (see :py:mod:`line_profiler.autoprofile.eager_preimports`), - regardless of whether those imports directly occur in the profiled - script/module/code. + are now: + + * Eagerly pre-imported to be profiled (see + :py:mod:`line_profiler.autoprofile.eager_preimports`), + regardless of whether those imports directly occur in the profiled + script/module/code. + * Descended/Recursed into if they are packages; pass + ``.__init__`` instead of ```` to curtail + descent and limit profiling to classes and functions in the local + namespace of the :file:`__init__.py`. + To restore the old behavior, pass the :option:`!--no-preimports` flag. """ @@ -124,7 +131,6 @@ def main(): import threading import asyncio # NOQA import concurrent.futures # NOQA -import pkgutil import tempfile import time import traceback From e57b7a68dd4827512ffce9868236023f6c0e04d4 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 27 May 2025 05:43:47 +0200 Subject: [PATCH 37/70] Separate `ScopingPolicy` into its own module CHANGELOG.rst Scrubbed reference to the removed `--recursive-prof-mod` flag docs/source/auto/line_profiler.rst Updated to add `line_profiler.scoping_policy` to the index docs/source/auto/line_profiler.scoping_policy.rst New boilerplate RST for the module line_profiler/scoping_policy.py[i] ScopingPolicy, ScopingPolicyDict, DEFAULT_SCOPING_POLICIES Migrated from `line_profiler/line_profiler.py` --- CHANGELOG.rst | 2 +- docs/source/auto/line_profiler.rst | 1 + .../auto/line_profiler.scoping_policy.rst | 8 + line_profiler/line_profiler.py | 333 +----------------- line_profiler/line_profiler.pyi | 52 +-- line_profiler/scoping_policy.py | 311 ++++++++++++++++ line_profiler/scoping_policy.pyi | 49 +++ 7 files changed, 385 insertions(+), 371 deletions(-) create mode 100644 docs/source/auto/line_profiler.scoping_policy.rst create mode 100644 line_profiler/scoping_policy.py create mode 100644 line_profiler/scoping_policy.pyi diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ea79d10d..6b96a08e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -15,7 +15,7 @@ Changes * ENH: In Python >=3.11, profiled objects are reported using their qualified name. * ENH: Highlight final summary using rich if enabled * ENH: Made it possible to use multiple profiler instances simultaneously -* ENH: ``kernprof --prof-mod`` target entities are now imported and profiled regardless of whether they are directly imported in the run script/module/code (old behavior restored by passing ``--no-preimports``; recurse into packages with ``--recursive-prof-mod``); made on-import profiling more aggressive so that it doesn't miss entities like class methods and properties +* ENH: ``kernprof --prof-mod`` target entities are now imported and profiled regardless of whether they are directly imported in the run script/module/code (old behavior recoed by passing ``--no-preimports``); made on-import profiling more aggressive so that it doesn't miss entities like class methods and properties 4.2.0 ~~~~~ diff --git a/docs/source/auto/line_profiler.rst b/docs/source/auto/line_profiler.rst index 4cdaccef..5e5cba62 100644 --- a/docs/source/auto/line_profiler.rst +++ b/docs/source/auto/line_profiler.rst @@ -18,6 +18,7 @@ Submodules line_profiler.__main__ line_profiler._line_profiler line_profiler.explicit_profiler + line_profiler.scoping_policy line_profiler.ipython_extension line_profiler.line_profiler diff --git a/docs/source/auto/line_profiler.scoping_policy.rst b/docs/source/auto/line_profiler.scoping_policy.rst new file mode 100644 index 00000000..20ce4822 --- /dev/null +++ b/docs/source/auto/line_profiler.scoping_policy.rst @@ -0,0 +1,8 @@ +line\_profiler.scoping\_policy module +===================================== + +.. automodule:: line_profiler.scoping_policy + :members: + :undoc-members: + :show-inheritance: + :private-members: diff --git a/line_profiler/line_profiler.py b/line_profiler/line_profiler.py index 0ac63887..9f10f42b 100755 --- a/line_profiler/line_profiler.py +++ b/line_profiler/line_profiler.py @@ -14,8 +14,6 @@ import types import warnings from argparse import ArgumentError, ArgumentParser -from enum import auto -from typing import Union, TypedDict try: from ._line_profiler import LineProfiler as CLineProfiler @@ -24,11 +22,11 @@ 'The line_profiler._line_profiler c-extension is not importable. ' f'Has it been compiled? Underlying error is ex={ex!r}' ) -from .line_profiler_utils import StringEnum from .profiler_mixin import (ByCountProfilerMixin, is_property, is_cached_property, is_boundmethod, is_classmethod, is_staticmethod, is_partial, is_partialmethod) +from .scoping_policy import ScopingPolicy # NOTE: This needs to be in sync with ../kernprof.py and __init__.py @@ -43,17 +41,6 @@ types.MethodWrapperType, types.WrapperDescriptorType) -#: Default scoping policies: -#: -#: * Profile sibling and descendant functions -#: (:py:attr:`ScopingPolicy.SIBLINGS`) -#: * Descend ingo sibling and descendant classes -#: (:py:attr:`ScopingPolicy.SIBLINGS`) -#: * Don't descend into modules (:py:attr:`ScopingPolicy.EXACT`) -DEFAULT_SCOPING_POLICIES = types.MappingProxyType({'func': 'siblings', - 'class': 'siblings', - 'module': 'exact'}) - is_function = inspect.isfunction @@ -119,302 +106,6 @@ def __init__(self, func, profiler_id): self.profiler_id = profiler_id -class ScopingPolicy(StringEnum): - """ - :py:class:`StrEnum` for scoping policies, that is, how it is - decided whether to: - - * Profile a function found in a namespace (a class or a module), and - * Descend into nested namespaces so that their methods and functions - are profiled, - - when using :py:meth:`LineProfiler.add_class`, - :py:meth:`LineProfiler.add_module`, and - :py:func:`~.add_imported_function_or_module()`. - - Available policies are: - - :py:attr:`ScopingPolicy.EXACT` - Only profile *functions* found in the namespace fulfilling - :py:attr:`ScopingPolicy.CHILDREN` as defined below, without - descending into nested namespaces - - :py:attr:`ScopingPolicy.CHILDREN` - Only profile/descend into *child* objects, which are: - - * Classes and functions defined *locally* in the very - module, or in the very class as its "inner classes" and - methods - * Direct submodules, in case when the namespace is a module - object representing a package - - :py:attr:`ScopingPolicy.DESCENDANTS` - Only profile/descend into *descendant* objects, which are: - - * Child classes, functions, and modules, as defined above in - :py:attr:`ScopingPolicy.CHILDREN` - * Their child classes, functions, and modules, ... - * ... and so on - - Note: - Since imported submodule module objects are by default - placed into the namespace of their parent-package module - objects, this functions largely identical to - :py:attr:`ScopingPolicy.CHILDREN` for descent from module - objects into other modules objects. - - :py:attr:`ScopingPolicy.SIBLINGS` - Only profile/descend into *sibling* and descendant objects, - which are: - - * Descendant classes, functions, and modules, as defined above - in :py:attr:`ScopingPolicy.DESCENDANTS` - * Classes and functions (and descendants thereof) defined in the - same parent namespace to this very class, or in modules (and - subpackages and their descendants) sharing a parent package - to this very module - * Modules (and subpackages and their descendants) sharing a - parent package, when the namespace is a module - - :py:attr:`ScopingPolicy.NONE` - Don't check scopes; profile all functions found in the local - namespace of the class/module, and descend into all nested - namespaces recursively - - Note: - This is probably a *very* bad idea for module scoping, - potentially resulting in accidentally recursing through a - significant portion of loaded modules; - proceed with care. - - Note: - Other than :py:class:`enum.Enum` methods starting and ending - with single underscores (e.g. :py:meth:`!_missing_`), all - methods prefixed with a single underscore are to be considered - implementation details. - """ - EXACT = auto() - CHILDREN = auto() - DESCENDANTS = auto() - SIBLINGS = auto() - NONE = auto() - - # Verification - - def __init_subclass__(cls, *args, **kwargs): - """ - Call :py:meth:`_check_class`. - """ - super().__init_subclass__(*args, **kwargs) - cls._check_class() - - @classmethod - def _check_class(cls): - """ - Verify that :py:meth:`.get_filter` return a callable for all - policy values and object types. - """ - mock_module = types.ModuleType('mock_module') - - class MockClass: - pass - - for member in cls.__members__.values(): - for obj_type in 'func', 'class', 'module': - for namespace in mock_module, MockClass: - assert callable(member.get_filter(namespace, obj_type)) - - # Filtering - - def get_filter(self, namespace, obj_type): - """ - Args: - namespace (Union[type, types.ModuleType]): - Class or module to be profiled. - obj_type (Literal['func', 'class', 'module']): - Type of object encountered in ``namespace``: - - ``'func'`` - Either a function, or a component function of a - callable-like object (e.g. :py:class:`property`) - - ``'class'`` (resp. ``'module'``) - A class (resp. a module) - - Returns: - func (Callable[..., bool]): - Filter callable returning whether the argument (as - specified by ``obj_type``) should be added - via :py:meth:`LineProfiler.add_class`, - :py:meth:`LineProfiler.add_module`, or - :py:meth:`LineProfiler.add_callable` - """ - is_class = isinstance(namespace, type) - if obj_type == 'module': - if is_class: - return self._return_const(False) - return self._get_module_filter_in_module(namespace) - if is_class: - method = self._get_callable_filter_in_class - else: - method = self._get_callable_filter_in_module - return method(namespace, is_class=(obj_type == 'class')) - - @classmethod - def to_policies(cls, policies=None): - """ - Normalize ``policies`` into a dictionary of policies for various - object types. - - Args: - policies (Union[str, ScopingPolicy, \ -ScopingPolicyDict, None]): - :py:class:`ScopingPolicy`, string convertible thereto - (case-insensitive), or a mapping containing such values - and the keys as outlined in the return value; - the default :py:const:`None` is equivalent to - :py:data:`DEFAULT_SCOPING_POLICIES`. - - Returns: - normalized_policies (dict[Literal['func', 'class', \ -'module'], ScopingPolicy]): - Dictionary with the following key-value pairs: - - ``'func'`` - :py:class:`ScopingPolicy` for profiling functions - and other callable-like objects composed thereof - (e.g. :py:class:`property`). - - ``'class'`` - :py:class:`ScopingPolicy` for descending into - classes. - - ``'module'`` - :py:class:`ScopingPolicy` for descending into - modules (if the namespace is itself a module). - - Note: - If ``policies`` is a mapping, it is required to contain all - three of the aforementioned keys. - - Example: - - >>> assert (ScopingPolicy.to_policies('children') - ... == dict.fromkeys(['func', 'class', 'module'], - ... ScopingPolicy.CHILDREN)) - >>> assert (ScopingPolicy.to_policies({ - ... 'func': 'NONE', - ... 'class': 'descendants', - ... 'module': 'exact', - ... 'unused key': 'unused value'}) - ... == {'func': ScopingPolicy.NONE, - ... 'class': ScopingPolicy.DESCENDANTS, - ... 'module': ScopingPolicy.EXACT}) - >>> ScopingPolicy.to_policies({}) - Traceback (most recent call last): - ... - KeyError: 'func' - """ - if policies is None: - policies = DEFAULT_SCOPING_POLICIES - if isinstance(policies, str): - policy = cls(policies) - return _ScopingPolicyDict( - dict.fromkeys(['func', 'class', 'module'], policy)) - return _ScopingPolicyDict({'func': cls(policies['func']), - 'class': cls(policies['class']), - 'module': cls(policies['module'])}) - - @staticmethod - def _return_const(value): - def return_const(*_, **__): - return value - - return return_const - - @staticmethod - def _match_prefix(s, prefix, sep='.'): - return s == prefix or s.startswith(prefix + sep) - - def _get_callable_filter_in_class(self, cls, is_class): - def func_is_child(other): - if not modules_are_equal(other): - return False - return other.__qualname__ == f'{cls.__qualname__}.{other.__name__}' - - def modules_are_equal(other): # = sibling check - return cls.__module__ == other.__module__ - - def func_is_descdendant(other): - if not modules_are_equal(other): - return False - return other.__qualname__.startswith(cls.__qualname__ + '.') - - return {'exact': (self._return_const(False) - if is_class else - func_is_child), - 'children': func_is_child, - 'descendants': func_is_descdendant, - 'siblings': modules_are_equal, - 'none': self._return_const(True)}[self.value] - - def _get_callable_filter_in_module(self, mod, is_class): - def func_is_child(other): - return other.__module__ == mod.__name__ - - def func_is_descdendant(other): - return self._match_prefix(other.__module__, mod.__name__) - - def func_is_cousin(other): - if func_is_descdendant(other): - return True - return self._match_prefix(other.__module__, parent) - - parent, _, basename = mod.__name__.rpartition('.') - return {'exact': (self._return_const(False) - if is_class else - func_is_child), - 'children': func_is_child, - 'descendants': func_is_descdendant, - 'siblings': (func_is_cousin # Only if a pkg - if basename else - func_is_descdendant), - 'none': self._return_const(True)}[self.value] - - def _get_module_filter_in_module(self, mod): - def module_is_descendant(other): - return other.__name__.startswith(mod.__name__ + '.') - - def module_is_child(other): - return other.__name__.rpartition('.')[0] == mod.__name__ - - def module_is_sibling(other): - return other.__name__.startswith(parent + '.') - - parent, _, basename = mod.__name__.rpartition('.') - return {'exact': self._return_const(False), - 'children': module_is_child, - 'descendants': module_is_descendant, - 'siblings': (module_is_sibling # Only if a pkg - if basename else - self._return_const(False)), - 'none': self._return_const(True)}[self.value] - - -# Sanity check in case we extended `ScopingPolicy` and forgot to update -# the corresponding methods -ScopingPolicy._check_class() - -ScopingPolicyDict = TypedDict('ScopingPolicyDict', - {'func': Union[str, ScopingPolicy], - 'class': Union[str, ScopingPolicy], - 'module': Union[str, ScopingPolicy]}) -_ScopingPolicyDict = TypedDict('_ScopingPolicyDict', - {'func': ScopingPolicy, - 'class': ScopingPolicy, - 'module': ScopingPolicy}) - - class LineProfiler(CLineProfiler, ByCountProfilerMixin): """ A profiler that records the execution times of individual lines. @@ -583,8 +274,8 @@ def add_class(self, cls, *, scoping_policy=None, wrap=False): Whether (and how) to match the scope of members and decide on whether to add them: - :py:class:`str` (incl. :py:class:`ScopingPolicy`): - Strings are converted to :py:class:`ScopingPolicy` + :py:class:`str` (incl. :py:class:`~.ScopingPolicy`): + Strings are converted to :py:class:`~.ScopingPolicy` instances in a case-insensitive manner, and the same policy applies to all members. @@ -594,10 +285,11 @@ def add_class(self, cls, *, scoping_policy=None, wrap=False): :py:const:`None` The default, equivalent to - :py:data:`DEFAULT_SCOPING_POLICIES`. + :py:data:\ +`~.scoping_policy.DEFAULT_SCOPING_POLICIES`. - See :py:class:`ScopingPolicy` and - :py:meth:`~ScopingPolicy.to_policies` for details. + See :py:class:`~.ScopingPolicy` and + :py:meth:`.ScopingPolicy.to_policies` for details. wrap (bool): Whether to replace the wrapped members with wrappers which automatically enable/disable the profiler when @@ -627,8 +319,8 @@ def add_module(self, mod, *, scoping_policy=None, wrap=False): Whether (and how) to match the scope of members and decide on whether to add them: - :py:class:`str` (incl. :py:class:`ScopingPolicy`): - Strings are converted to :py:class:`ScopingPolicy` + :py:class:`str` (incl. :py:class:`~.ScopingPolicy`): + Strings are converted to :py:class:`~.ScopingPolicy` instances in a case-insensitive manner, and the same policy applies to all members. @@ -638,10 +330,11 @@ def add_module(self, mod, *, scoping_policy=None, wrap=False): :py:const:`None` The default, equivalent to - :py:data:`DEFAULT_SCOPING_POLICIES`. + :py:data:\ +`~.scoping_policy.DEFAULT_SCOPING_POLICIES`. - See :py:class:`ScopingPolicy` and - :py:meth:`~ScopingPolicy.to_policies` for details. + See :py:class:`~.ScopingPolicy` and + :py:meth:`.ScopingPolicy.to_policies` for details. wrap (bool): Whether to replace the wrapped members with wrappers which automatically enable/disable the profiler when diff --git a/line_profiler/line_profiler.pyi b/line_profiler/line_profiler.pyi index 7502bab8..b943255d 100644 --- a/line_profiler/line_profiler.pyi +++ b/line_profiler/line_profiler.pyi @@ -1,13 +1,10 @@ import io -from enum import auto from functools import cached_property, partial, partialmethod -from inspect import isfunction as is_function from types import (FunctionType, MethodType, ModuleType, BuiltinFunctionType, BuiltinMethodType, ClassMethodDescriptorType, MethodDescriptorType, MethodWrapperType, WrapperDescriptorType) -from typing import (overload, - Any, Callable, List, Literal, Tuple, TypeVar, TypedDict) +from typing import overload, Any, Callable, List, Literal, Tuple, TypeVar try: from typing import ( # type: ignore[attr-defined] # noqa: F401 TypeIs) @@ -15,8 +12,8 @@ except ImportError: # Python < 3.13 from typing_extensions import TypeIs # noqa: F401 from _typeshed import Incomplete from ._line_profiler import LineProfiler as CLineProfiler -from .line_profiler_utils import StringEnum from .profiler_mixin import ByCountProfilerMixin +from .scoping_policy import ScopingPolicy, ScopingPolicyDict CLevelCallable = TypeVar('CLevelCallable', @@ -36,51 +33,6 @@ def load_ipython_extension(ip) -> None: ... -class ScopingPolicy(StringEnum): - CHILDREN = auto() - DESCENDANTS = auto() - SIBLINGS = auto() - NONE = auto() - - @overload - def get_filter( - self, - namespace: type | ModuleType, - obj_type: Literal['func']) -> Callable[[FunctionType], bool]: - ... - - @overload - def get_filter( - self, - namespace: type | ModuleType, - obj_type: Literal['class']) -> Callable[[type], bool]: - ... - - @overload - def get_filter( - self, - namespace: type | ModuleType, - obj_type: Literal['module']) -> Callable[[ModuleType], bool]: - ... - - @classmethod - def to_policies( - cls, - policies: (str | 'ScopingPolicy' - | ScopingPolicyDict | None) = None) -> _ScopingPolicyDict: - ... - - -ScopingPolicyDict = TypedDict('ScopingPolicyDict', - {'func': str | ScopingPolicy, - 'class': str | ScopingPolicy, - 'module': str | ScopingPolicy}) -_ScopingPolicyDict = TypedDict('_ScopingPolicyDict', - {'func': str | ScopingPolicy, - 'class': str | ScopingPolicy, - 'module': str | ScopingPolicy}) - - class LineProfiler(CLineProfiler, ByCountProfilerMixin): @overload def __call__(self, # type: ignore[overload-overlap] diff --git a/line_profiler/scoping_policy.py b/line_profiler/scoping_policy.py new file mode 100644 index 00000000..cedf51e9 --- /dev/null +++ b/line_profiler/scoping_policy.py @@ -0,0 +1,311 @@ +from enum import auto +from types import MappingProxyType, ModuleType +from typing import Union, TypedDict +from .line_profiler_utils import StringEnum + + +#: Default scoping policies: +#: +#: * Profile sibling and descendant functions +#: (:py:attr:`ScopingPolicy.SIBLINGS`) +#: * Descend ingo sibling and descendant classes +#: (:py:attr:`ScopingPolicy.SIBLINGS`) +#: * Don't descend into modules (:py:attr:`ScopingPolicy.EXACT`) +DEFAULT_SCOPING_POLICIES = MappingProxyType( + {'func': 'siblings', 'class': 'siblings', 'module': 'exact'}) + + +class ScopingPolicy(StringEnum): + """ + :py:class:`StrEnum` for scoping policies, that is, how it is + decided whether to: + + * Profile a function found in a namespace (a class or a module), and + * Descend into nested namespaces so that their methods and functions + are profiled, + + when using :py:meth:`LineProfiler.add_class`, + :py:meth:`LineProfiler.add_module`, and + :py:func:`~.add_imported_function_or_module()`. + + Available policies are: + + :py:attr:`ScopingPolicy.EXACT` + Only profile *functions* found in the namespace fulfilling + :py:attr:`ScopingPolicy.CHILDREN` as defined below, without + descending into nested namespaces + + :py:attr:`ScopingPolicy.CHILDREN` + Only profile/descend into *child* objects, which are: + + * Classes and functions defined *locally* in the very + module, or in the very class as its "inner classes" and + methods + * Direct submodules, in case when the namespace is a module + object representing a package + + :py:attr:`ScopingPolicy.DESCENDANTS` + Only profile/descend into *descendant* objects, which are: + + * Child classes, functions, and modules, as defined above in + :py:attr:`ScopingPolicy.CHILDREN` + * Their child classes, functions, and modules, ... + * ... and so on + + Note: + Since imported submodule module objects are by default + placed into the namespace of their parent-package module + objects, this functions largely identical to + :py:attr:`ScopingPolicy.CHILDREN` for descent from module + objects into other modules objects. + + :py:attr:`ScopingPolicy.SIBLINGS` + Only profile/descend into *sibling* and descendant objects, + which are: + + * Descendant classes, functions, and modules, as defined above + in :py:attr:`ScopingPolicy.DESCENDANTS` + * Classes and functions (and descendants thereof) defined in the + same parent namespace to this very class, or in modules (and + subpackages and their descendants) sharing a parent package + to this very module + * Modules (and subpackages and their descendants) sharing a + parent package, when the namespace is a module + + :py:attr:`ScopingPolicy.NONE` + Don't check scopes; profile all functions found in the local + namespace of the class/module, and descend into all nested + namespaces recursively + + Note: + This is probably a *very* bad idea for module scoping, + potentially resulting in accidentally recursing through a + significant portion of loaded modules; + proceed with care. + + Note: + Other than :py:class:`enum.Enum` methods starting and ending + with single underscores (e.g. :py:meth:`!_missing_`), all + methods prefixed with a single underscore are to be considered + implementation details. + """ + EXACT = auto() + CHILDREN = auto() + DESCENDANTS = auto() + SIBLINGS = auto() + NONE = auto() + + # Verification + + def __init_subclass__(cls, *args, **kwargs): + """ + Call :py:meth:`_check_class`. + """ + super().__init_subclass__(*args, **kwargs) + cls._check_class() + + @classmethod + def _check_class(cls): + """ + Verify that :py:meth:`.get_filter` return a callable for all + policy values and object types. + """ + mock_module = ModuleType('mock_module') + + class MockClass: + pass + + for member in cls.__members__.values(): + for obj_type in 'func', 'class', 'module': + for namespace in mock_module, MockClass: + assert callable(member.get_filter(namespace, obj_type)) + + # Filtering + + def get_filter(self, namespace, obj_type): + """ + Args: + namespace (Union[type, types.ModuleType]): + Class or module to be profiled. + obj_type (Literal['func', 'class', 'module']): + Type of object encountered in ``namespace``: + + ``'func'`` + Either a function, or a component function of a + callable-like object (e.g. :py:class:`property`) + + ``'class'`` (resp. ``'module'``) + A class (resp. a module) + + Returns: + func (Callable[..., bool]): + Filter callable returning whether the argument (as + specified by ``obj_type``) should be added + via :py:meth:`LineProfiler.add_class`, + :py:meth:`LineProfiler.add_module`, or + :py:meth:`LineProfiler.add_callable` + """ + is_class = isinstance(namespace, type) + if obj_type == 'module': + if is_class: + return self._return_const(False) + return self._get_module_filter_in_module(namespace) + if is_class: + method = self._get_callable_filter_in_class + else: + method = self._get_callable_filter_in_module + return method(namespace, is_class=(obj_type == 'class')) + + @classmethod + def to_policies(cls, policies=None): + """ + Normalize ``policies`` into a dictionary of policies for various + object types. + + Args: + policies (Union[str, ScopingPolicy, \ +ScopingPolicyDict, None]): + :py:class:`ScopingPolicy`, string convertible thereto + (case-insensitive), or a mapping containing such values + and the keys as outlined in the return value; + the default :py:const:`None` is equivalent to + :py:data:`DEFAULT_SCOPING_POLICIES`. + + Returns: + normalized_policies (dict[Literal['func', 'class', \ +'module'], ScopingPolicy]): + Dictionary with the following key-value pairs: + + ``'func'`` + :py:class:`ScopingPolicy` for profiling functions + and other callable-like objects composed thereof + (e.g. :py:class:`property`). + + ``'class'`` + :py:class:`ScopingPolicy` for descending into + classes. + + ``'module'`` + :py:class:`ScopingPolicy` for descending into + modules (if the namespace is itself a module). + + Note: + If ``policies`` is a mapping, it is required to contain all + three of the aforementioned keys. + + Example: + + >>> assert (ScopingPolicy.to_policies('children') + ... == dict.fromkeys(['func', 'class', 'module'], + ... ScopingPolicy.CHILDREN)) + >>> assert (ScopingPolicy.to_policies({ + ... 'func': 'NONE', + ... 'class': 'descendants', + ... 'module': 'exact', + ... 'unused key': 'unused value'}) + ... == {'func': ScopingPolicy.NONE, + ... 'class': ScopingPolicy.DESCENDANTS, + ... 'module': ScopingPolicy.EXACT}) + >>> ScopingPolicy.to_policies({}) + Traceback (most recent call last): + ... + KeyError: 'func' + """ + if policies is None: + policies = DEFAULT_SCOPING_POLICIES + if isinstance(policies, str): + policy = cls(policies) + return _ScopingPolicyDict( + dict.fromkeys(['func', 'class', 'module'], policy)) + return _ScopingPolicyDict({'func': cls(policies['func']), + 'class': cls(policies['class']), + 'module': cls(policies['module'])}) + + @staticmethod + def _return_const(value): + def return_const(*_, **__): + return value + + return return_const + + @staticmethod + def _match_prefix(s, prefix, sep='.'): + return s == prefix or s.startswith(prefix + sep) + + def _get_callable_filter_in_class(self, cls, is_class): + def func_is_child(other): + if not modules_are_equal(other): + return False + return other.__qualname__ == f'{cls.__qualname__}.{other.__name__}' + + def modules_are_equal(other): # = sibling check + return cls.__module__ == other.__module__ + + def func_is_descdendant(other): + if not modules_are_equal(other): + return False + return other.__qualname__.startswith(cls.__qualname__ + '.') + + return {'exact': (self._return_const(False) + if is_class else + func_is_child), + 'children': func_is_child, + 'descendants': func_is_descdendant, + 'siblings': modules_are_equal, + 'none': self._return_const(True)}[self.value] + + def _get_callable_filter_in_module(self, mod, is_class): + def func_is_child(other): + return other.__module__ == mod.__name__ + + def func_is_descdendant(other): + return self._match_prefix(other.__module__, mod.__name__) + + def func_is_cousin(other): + if func_is_descdendant(other): + return True + return self._match_prefix(other.__module__, parent) + + parent, _, basename = mod.__name__.rpartition('.') + return {'exact': (self._return_const(False) + if is_class else + func_is_child), + 'children': func_is_child, + 'descendants': func_is_descdendant, + 'siblings': (func_is_cousin # Only if a pkg + if basename else + func_is_descdendant), + 'none': self._return_const(True)}[self.value] + + def _get_module_filter_in_module(self, mod): + def module_is_descendant(other): + return other.__name__.startswith(mod.__name__ + '.') + + def module_is_child(other): + return other.__name__.rpartition('.')[0] == mod.__name__ + + def module_is_sibling(other): + return other.__name__.startswith(parent + '.') + + parent, _, basename = mod.__name__.rpartition('.') + return {'exact': self._return_const(False), + 'children': module_is_child, + 'descendants': module_is_descendant, + 'siblings': (module_is_sibling # Only if a pkg + if basename else + self._return_const(False)), + 'none': self._return_const(True)}[self.value] + + +# Sanity check in case we extended `ScopingPolicy` and forgot to update +# the corresponding methods +ScopingPolicy._check_class() + +ScopingPolicyDict = TypedDict('ScopingPolicyDict', + {'func': Union[str, ScopingPolicy], + 'class': Union[str, ScopingPolicy], + 'module': Union[str, ScopingPolicy]}) +_ScopingPolicyDict = TypedDict('_ScopingPolicyDict', + {'func': ScopingPolicy, + 'class': ScopingPolicy, + 'module': ScopingPolicy}) diff --git a/line_profiler/scoping_policy.pyi b/line_profiler/scoping_policy.pyi new file mode 100644 index 00000000..0a45a09b --- /dev/null +++ b/line_profiler/scoping_policy.pyi @@ -0,0 +1,49 @@ +from enum import auto +from types import FunctionType, ModuleType +from typing import overload, Literal, Callable, TypedDict +from .line_profiler_utils import StringEnum + + +class ScopingPolicy(StringEnum): + CHILDREN = auto() + DESCENDANTS = auto() + SIBLINGS = auto() + NONE = auto() + + @overload + def get_filter( + self, + namespace: type | ModuleType, + obj_type: Literal['func']) -> Callable[[FunctionType], bool]: + ... + + @overload + def get_filter( + self, + namespace: type | ModuleType, + obj_type: Literal['class']) -> Callable[[type], bool]: + ... + + @overload + def get_filter( + self, + namespace: type | ModuleType, + obj_type: Literal['module']) -> Callable[[ModuleType], bool]: + ... + + @classmethod + def to_policies( + cls, + policies: (str | 'ScopingPolicy' | 'ScopingPolicyDict' + | None) = None) -> '_ScopingPolicyDict': + ... + + +ScopingPolicyDict = TypedDict('ScopingPolicyDict', + {'func': str | ScopingPolicy, + 'class': str | ScopingPolicy, + 'module': str | ScopingPolicy}) +_ScopingPolicyDict = TypedDict('_ScopingPolicyDict', + {'func': str | ScopingPolicy, + 'class': str | ScopingPolicy, + 'module': str | ScopingPolicy}) From f6d84c5ab4615dfa9ad50e61e637ca2b8e8cf240 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 27 May 2025 08:22:34 +0200 Subject: [PATCH 38/70] Better support for handling classes line_profiler/autoprofile/line_profiler_utils.pyi Rerouted imports to their updated locations line_profiler/line_profiler.py[i] Removed (now) unused imports LineProfiler add_callable() Refactored to use the new `.get_underlying_functions()` _add_namespace() Refactored to use the new `._wrap_namespace_members()` to handle `wrap = True` line_profiler/profiler_mixin.py[i] Added missing type hints C_LEVEL_CALLABLE_TYPES, is_function(), is_c_level_callable() Moved from `line_profiler/line_profiler.py` ByCountProfilerMixin wrap_callable() - Added dispatch of classes to the new `.wrap_class()` - Added check against non-callables, raising `TypeError` get_underlying_functions() - Refactored from `line_profiler/line_profiler.py:: _get_underlying_functions()` - Added handling for classes - Added check against recursion (like `LineProfiler.add_class()` and `LineProfiler.add_module()` has) - Added check against `.__call__()` being a C-level callable wrap_class() New method for wrapping a class and all its wrappable locally-defined members _wrap_namespace_members() Refactored out shared code between `.wrap_class()` and `LineProfiler._add_namespace()` --- .../autoprofile/line_profiler_utils.pyi | 7 +- line_profiler/line_profiler.py | 84 +------- line_profiler/line_profiler.pyi | 25 +-- line_profiler/profiler_mixin.py | 183 +++++++++++++++--- line_profiler/profiler_mixin.pyi | 96 ++++++--- 5 files changed, 245 insertions(+), 150 deletions(-) diff --git a/line_profiler/autoprofile/line_profiler_utils.pyi b/line_profiler/autoprofile/line_profiler_utils.pyi index a1116cbe..b710b577 100644 --- a/line_profiler/autoprofile/line_profiler_utils.pyi +++ b/line_profiler/autoprofile/line_profiler_utils.pyi @@ -2,8 +2,9 @@ from types import ModuleType from typing import overload, Any, Literal, TYPE_CHECKING if TYPE_CHECKING: # Stub-only annotations - from ..line_profiler import (CLevelCallable, CallableLike, - ScopingPolicy, ScopingPolicyDict) + from ..line_profiler import CallableLike + from ..profiler_mixin import CLevelCallable + from ..scoping_policy import ScopingPolicy, ScopingPolicyDict @overload @@ -17,7 +18,7 @@ def add_imported_function_or_module( @overload def add_imported_function_or_module( - self, item: CallableLike | type | ModuleType, + self, item: CallableLike | ModuleType, scoping_policy: ( ScopingPolicy | str | ScopingPolicyDict | None) = None, wrap: bool = False) -> Literal[0, 1]: diff --git a/line_profiler/line_profiler.py b/line_profiler/line_profiler.py index 9f10f42b..e2c209a3 100755 --- a/line_profiler/line_profiler.py +++ b/line_profiler/line_profiler.py @@ -12,7 +12,6 @@ import sys import tempfile import types -import warnings from argparse import ArgumentError, ArgumentParser try: @@ -22,37 +21,13 @@ 'The line_profiler._line_profiler c-extension is not importable. ' f'Has it been compiled? Underlying error is ex={ex!r}' ) -from .profiler_mixin import (ByCountProfilerMixin, - is_property, is_cached_property, - is_boundmethod, is_classmethod, is_staticmethod, - is_partial, is_partialmethod) +from .profiler_mixin import ByCountProfilerMixin, is_c_level_callable from .scoping_policy import ScopingPolicy # NOTE: This needs to be in sync with ../kernprof.py and __init__.py __version__ = '4.3.0' -# These objects are callables, but are defined in C so we can't handle -# them anyway -C_LEVEL_CALLABLE_TYPES = (types.BuiltinFunctionType, - types.BuiltinMethodType, - types.ClassMethodDescriptorType, - types.MethodDescriptorType, - types.MethodWrapperType, - types.WrapperDescriptorType) - -is_function = inspect.isfunction - - -def is_c_level_callable(func): - """ - Returns: - func_is_c_level (bool): - Whether a callable is defined at the C level (and is thus - non-profilable). - """ - return isinstance(func, C_LEVEL_CALLABLE_TYPES) - def load_ipython_extension(ip): """ API for IPython to recognize this module as an IPython extension. @@ -61,36 +36,6 @@ def load_ipython_extension(ip): ip.register_magics(LineProfilerMagics) -def _get_underlying_functions(func): - """ - Get the underlying function objects of a callable or an adjacent - object. - - Returns: - funcs (list[Callable]) - """ - if any(check(func) - for check in (is_boundmethod, is_classmethod, is_staticmethod)): - return _get_underlying_functions(func.__func__) - if any(check(func) - for check in (is_partial, is_partialmethod, is_cached_property)): - return _get_underlying_functions(func.func) - if is_property(func): - result = [] - for impl in func.fget, func.fset, func.fdel: - if impl is not None: - result.extend(_get_underlying_functions(impl)) - return result - if not callable(func): - raise TypeError(f'func = {func!r}: ' - f'cannot get functions from {type(func)} objects') - if is_function(func): - return [func] - if is_c_level_callable(func): - return [] - return [type(func).__call__] - - class _WrapperInfo: """ Helper object for holding the state of a wrapper function. @@ -131,7 +76,8 @@ def __call__(self, func): """ # The same object is returned when: # - `func` is a `types.FunctionType` which is already - # decorated by the profiler, or + # decorated by the profiler, + # - `func` is a class, or # - `func` is any of the C-level callables that can't be # profiled # otherwise, wrapper objects are always returned. @@ -169,7 +115,7 @@ def add_callable(self, func, guard=None): guard = self._already_a_wrapper nadded = 0 - for impl in _get_underlying_functions(func): + for impl in self.get_underlying_functions(func): info, wrapped_by_this_prof = self._get_wrapper_info(impl) if wrapped_by_this_prof if guard is None else guard(impl): continue @@ -219,7 +165,7 @@ def func_guard(func): class_scoping_policy=class_scoping_policy, module_scoping_policy=module_scoping_policy, wrap=wrap) - wrap_failures = {} + members_to_wrap = {} func_check = func_scoping_policy.get_filter(namespace, 'func') cls_check = class_scoping_policy.get_filter(namespace, 'class') mod_check = module_scoping_policy.get_filter(namespace, 'module') @@ -242,23 +188,11 @@ def func_guard(func): except TypeError: # Not a callable (wrapper) continue if wrap: - wrapper = self.wrap_callable(value) - if wrapper is not value: - try: - setattr(namespace, attr, wrapper) - except (TypeError, AttributeError): - # Corner case in case if a class/module don't - # allow setting attributes (could e.g. happen - # with some builtin/extension classes, but their - # method should be in C anyway, so - # `.add_callable()` should've returned 0 and we - # shouldn't be here) - wrap_failures[attr] = value + members_to_wrap[attr] = value count += 1 - if wrap_failures: - msg = (f'cannot wrap {len(wrap_failures)} attribute(s) of ' - f'{namespace!r} (`{{attr: value}}`): {wrap_failures!r}') - warnings.warn(msg, stacklevel=2) + if wrap and members_to_wrap: + self._wrap_namespace_members(namespace, members_to_wrap, + warning_stack_level=3) return count def add_class(self, cls, *, scoping_policy=None, wrap=False): diff --git a/line_profiler/line_profiler.pyi b/line_profiler/line_profiler.pyi index b943255d..f794e50f 100644 --- a/line_profiler/line_profiler.pyi +++ b/line_profiler/line_profiler.pyi @@ -1,32 +1,17 @@ import io from functools import cached_property, partial, partialmethod -from types import (FunctionType, MethodType, ModuleType, - BuiltinFunctionType, BuiltinMethodType, - ClassMethodDescriptorType, MethodDescriptorType, - MethodWrapperType, WrapperDescriptorType) -from typing import overload, Any, Callable, List, Literal, Tuple, TypeVar -try: - from typing import ( # type: ignore[attr-defined] # noqa: F401 - TypeIs) -except ImportError: # Python < 3.13 - from typing_extensions import TypeIs # noqa: F401 +from types import FunctionType, MethodType, ModuleType +from typing import overload, Callable, List, Literal, Tuple, TypeVar from _typeshed import Incomplete from ._line_profiler import LineProfiler as CLineProfiler -from .profiler_mixin import ByCountProfilerMixin +from .profiler_mixin import ByCountProfilerMixin, CLevelCallable from .scoping_policy import ScopingPolicy, ScopingPolicyDict -CLevelCallable = TypeVar('CLevelCallable', - BuiltinFunctionType, BuiltinMethodType, - ClassMethodDescriptorType, MethodDescriptorType, - MethodWrapperType, WrapperDescriptorType) CallableLike = TypeVar('CallableLike', FunctionType, partial, property, cached_property, - MethodType, staticmethod, classmethod, partialmethod) - - -def is_c_level_callable(func: Any) -> TypeIs[CLevelCallable]: - ... + MethodType, staticmethod, classmethod, partialmethod, + type) def load_ipython_extension(ip) -> None: diff --git a/line_profiler/profiler_mixin.py b/line_profiler/profiler_mixin.py index 52980765..0c155bd3 100644 --- a/line_profiler/profiler_mixin.py +++ b/line_profiler/profiler_mixin.py @@ -1,12 +1,34 @@ import functools import inspect import types +from warnings import warn +from .scoping_policy import ScopingPolicy is_coroutine = inspect.iscoroutinefunction +is_function = inspect.isfunction is_generator = inspect.isgeneratorfunction is_async_generator = inspect.isasyncgenfunction +# These objects are callables, but are defined in C so we can't handle +# them anyway +C_LEVEL_CALLABLE_TYPES = (types.BuiltinFunctionType, + types.BuiltinMethodType, + types.ClassMethodDescriptorType, + types.MethodDescriptorType, + types.MethodWrapperType, + types.WrapperDescriptorType) + + +def is_c_level_callable(func): + """ + Returns: + func_is_c_level (bool): + Whether a callable is defined at the C level (and is thus + non-profilable). + """ + return isinstance(func, C_LEVEL_CALLABLE_TYPES) + def is_classmethod(f): return isinstance(f, classmethod) @@ -50,28 +72,98 @@ def wrap_callable(self, func): it on function exit. """ if is_classmethod(func): - wrapper = self.wrap_classmethod(func) - elif is_staticmethod(func): - wrapper = self.wrap_staticmethod(func) - elif is_boundmethod(func): - wrapper = self.wrap_boundmethod(func) - elif is_partialmethod(func): - wrapper = self.wrap_partialmethod(func) - elif is_partial(func): - wrapper = self.wrap_partial(func) - elif is_property(func): - wrapper = self.wrap_property(func) - elif is_cached_property(func): - wrapper = self.wrap_cached_property(func) - elif is_async_generator(func): - wrapper = self.wrap_async_generator(func) - elif is_coroutine(func): - wrapper = self.wrap_coroutine(func) - elif is_generator(func): - wrapper = self.wrap_generator(func) - else: - wrapper = self.wrap_function(func) - return wrapper + return self.wrap_classmethod(func) + if is_staticmethod(func): + return self.wrap_staticmethod(func) + if is_boundmethod(func): + return self.wrap_boundmethod(func) + if is_partialmethod(func): + return self.wrap_partialmethod(func) + if is_partial(func): + return self.wrap_partial(func) + if is_property(func): + return self.wrap_property(func) + if is_cached_property(func): + return self.wrap_cached_property(func) + if is_async_generator(func): + return self.wrap_async_generator(func) + if is_coroutine(func): + return self.wrap_coroutine(func) + if is_generator(func): + return self.wrap_generator(func) + if isinstance(func, type): + return self.wrap_class(func) + if callable(func): + return self.wrap_function(func) + raise TypeError(f'func = {func!r}: does not look like a callable or ' + 'callable wrapper') + + @classmethod + def get_underlying_functions(cls, func): + """ + Get the underlying function objects of a callable or an adjacent + object. + + Returns: + funcs (list[Callable]) + """ + return cls._get_underlying_functions(func) + + @classmethod + def _get_underlying_functions(cls, func, seen=None, stop_at_classes=False): + if seen is None: + seen = set() + get_underlying = functools.partial( + cls._get_underlying_functions, + seen=seen, stop_at_classes=stop_at_classes) + if any(check(func) + for check in (is_boundmethod, is_classmethod, is_staticmethod)): + return get_underlying(func.__func__) + if any(check(func) + for check in (is_partial, is_partialmethod, is_cached_property)): + return get_underlying(func.func) + if is_property(func): + result = [] + for impl in func.fget, func.fset, func.fdel: + if impl is not None: + result.extend(get_underlying(impl)) + return result + if isinstance(func, type): + if stop_at_classes: + return [func] + result = [] + get_filter = cls._class_scoping_policy.get_filter + func_check = get_filter(func, 'func') + cls_check = get_filter(func, 'class') + for member in vars(func).values(): + try: + member_funcs = get_underlying(member, stop_at_classes=True) + except TypeError: + continue + for impl in member_funcs: + is_type = isinstance(impl, type) + check = cls_check if is_type else func_check + if not check(impl): + continue + if is_type: + result.extend(get_underlying(impl)) + else: + result.append(impl) + return result + if not callable(func): + raise TypeError(f'func = {func!r}: ' + f'cannot get functions from {type(func)} objects') + if id(func) in seen: + return [] + seen.add(id(func)) + if is_function(func): + return [func] + if is_c_level_callable(func): + return [] + func = type(func).__call__ + if is_c_level_callable(func): # Can happen with builtin types + return [] + return [func] def _wrap_callable_wrapper(self, wrapper, impl_attrs, *, args=None, kwargs=None, name_attr=None): @@ -281,6 +373,52 @@ def wrapper(*args, **kwds): return self._mark_wrapper(wrapper) + def wrap_class(self, func): + """ + Wrap a class by wrapping all locally-defined callables and + callable wrappers. + """ + get_filter = self._class_scoping_policy.get_filter + func_check = get_filter(func, 'func') + cls_check = get_filter(func, 'class') + get_underlying = functools.partial( + self._get_underlying_functions, stop_at_classes=True) + members_to_wrap = {} + for name, member in vars(func).items(): + try: + impls = get_underlying(member) + except TypeError: # Not a callable (wrapper) + continue + if any((cls_check(impl) + if isinstance(impl, type) else + func_check(impl)) + for impl in impls): + members_to_wrap[name] = member + self._wrap_namespace_members(func, members_to_wrap, + warning_stack_level=2) + return func + + def _wrap_namespace_members( + self, namespace, members, *, warning_stack_level=2): + wrap_failures = {} + for name, member in members.items(): + wrapper = self.wrap_callable(member) + if wrapper is member: + continue + try: + setattr(namespace, name, wrapper) + except (TypeError, AttributeError): + # Corner case in case if a class/module don't allow + # setting attributes (could e.g. happen with some + # builtin/extension classes, but their method should be + # in C anyway, so `.add_callable()` should've returned 0 + # and we shouldn't be here) + wrap_failures[name] = member + if wrap_failures: + msg = (f'cannot wrap {len(wrap_failures)} attribute(s) of ' + f'{namespace!r} (`{{attr: value}}`): {wrap_failures!r}') + warn(msg, stacklevel=warning_stack_level) + def _already_a_wrapper(self, func): return getattr(func, self._profiler_wrapped_marker, None) == id(self) @@ -322,3 +460,4 @@ def __exit__(self, *_, **__): self.disable_by_count() _profiler_wrapped_marker = '__line_profiler_id__' + _class_scoping_policy = ScopingPolicy.CHILDREN diff --git a/line_profiler/profiler_mixin.pyi b/line_profiler/profiler_mixin.pyi index 0f751b06..bce5453a 100644 --- a/line_profiler/profiler_mixin.pyi +++ b/line_profiler/profiler_mixin.pyi @@ -1,89 +1,125 @@ -import inspect -import typing - - -is_coroutine = inspect.iscoroutinefunction -is_generator = inspect.isgeneratorfunction -is_async_generator = inspect.isasyncgenfunction +from functools import cached_property, partial, partialmethod +from types import (FunctionType, MethodType, + BuiltinFunctionType, BuiltinMethodType, + ClassMethodDescriptorType, MethodDescriptorType, + MethodWrapperType, WrapperDescriptorType) +from typing import Any, Callable, Dict, List, Mapping, TypeVar +try: + from typing import ( # type: ignore[attr-defined] # noqa: F401 + ParamSpec) +except ImportError: # Python < 3.10 + from typing_extensions import ParamSpec # noqa: F401 +try: + from typing import ( # type: ignore[attr-defined] # noqa: F401 + Self) +except ImportError: # Python < 3.11 + from typing_extensions import Self # noqa: F401 +try: + from typing import ( # type: ignore[attr-defined] # noqa: F401 + TypeIs) +except ImportError: # Python < 3.13 + from typing_extensions import TypeIs # noqa: F401 + + +CLevelCallable = TypeVar('CLevelCallable', + BuiltinFunctionType, BuiltinMethodType, + ClassMethodDescriptorType, MethodDescriptorType, + MethodWrapperType, WrapperDescriptorType) +T = TypeVar('T', bound=type) +R = TypeVar('R') +PS = ParamSpec('PS') + + +def is_c_level_callable(func: Any) -> TypeIs[CLevelCallable]: + ... -def is_classmethod(f) -> bool: +def is_classmethod(f: Any) -> TypeIs[classmethod]: ... -def is_staticmethod(f) -> bool: +def is_staticmethod(f: Any) -> TypeIs[staticmethod]: ... -def is_boundmethod(f) -> bool: +def is_boundmethod(f: Any) -> TypeIs[MethodType]: ... -def is_partialmethod(f) -> bool: +def is_partialmethod(f: Any) -> TypeIs[partialmethod]: ... -def is_partial(f) -> bool: +def is_partial(f: Any) -> TypeIs[partial]: ... -def is_property(f) -> bool: +def is_property(f: Any) -> TypeIs[property]: ... -def is_cached_property(f) -> bool: +def is_cached_property(f: Any) -> TypeIs[cached_property]: ... class ByCountProfilerMixin: + def get_underlying_functions(self, func) -> List[FunctionType]: + ... def wrap_callable(self, func): ... - def wrap_classmethod(self, func): + def wrap_classmethod(self, func: classmethod) -> classmethod: + ... + + def wrap_staticmethod(self, func: staticmethod) -> staticmethod: ... - def wrap_staticmethod(self, func): + def wrap_boundmethod(self, func: MethodType) -> MethodType: ... - def wrap_boundmethod(self, func): + def wrap_partialmethod(self, func: partialmethod) -> partialmethod: ... - def wrap_partialmethod(self, func): + def wrap_partial(self, func: partial) -> partial: ... - def wrap_partial(self, func): + def wrap_property(self, func: property) -> property: ... - def wrap_property(self, func): + def wrap_cached_property(self, func: cached_property) -> cached_property: ... - def wrap_cached_property(self, func): + def wrap_async_generator(self, func: FunctionType) -> FunctionType: ... - def wrap_async_generator(self, func): + def wrap_coroutine(self, func: FunctionType) -> FunctionType: ... - def wrap_coroutine(self, func): + def wrap_generator(self, func: FunctionType) -> FunctionType: ... - def wrap_generator(self, func): + def wrap_function(self, func: Callable) -> FunctionType: ... - def wrap_function(self, func): + def wrap_class(self, func: T) -> T: ... - def run(self, cmd): + def run(self, cmd: str) -> Self: ... - def runctx(self, cmd, globals, locals): + def runctx(self, + cmd: str, + globals: Dict[str, Any] | None, + locals: Mapping[str, Any] | None) -> Self: ... - def runcall(self, func, /, *args, **kw): + def runcall(self, func: Callable[PS, R], /, + *args: PS.args, **kw: PS.kwargs) -> R: ... - def __enter__(self): + def __enter__(self) -> Self: ... - def __exit__(self, *_, **__): + def __exit__(self, *_, **__) -> None: ... From 1a975f4b7df3a26aaf8f90688ec56677ed14d9a0 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 27 May 2025 08:34:12 +0200 Subject: [PATCH 39/70] Fixed lint on stub file --- line_profiler/line_profiler_utils.pyi | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/line_profiler/line_profiler_utils.pyi b/line_profiler/line_profiler_utils.pyi index ad7447ff..a510cf94 100644 --- a/line_profiler/line_profiler_utils.pyi +++ b/line_profiler/line_profiler_utils.pyi @@ -1,5 +1,4 @@ import enum -from typing import Type, TypeVar, Union try: from typing import Self # type: ignore[attr-defined] # noqa: F401 except ImportError: # Python < 3.11 @@ -23,5 +22,5 @@ class StringEnum(str, enum.Enum): # type: ignore[misc] ... @classmethod - def _missing_(cls, value) -> Union[Self, None]: + def _missing_(cls, value) -> Self | None: ... From 4568db50fce436c1388cf95e97561ba06b14f200 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 27 May 2025 09:16:25 +0200 Subject: [PATCH 40/70] Doc for `.profiler_mixin` docs/source/auto/line_profiler.profiler_mixin.rst New boilerplate RST so that `line_profiler.profiler_mixin` is indexed and built as a doc page docs/source/auto/line_profiler.rst Added the above to the index line_profiler/profiler_mixin.py Updated docstrings to be more `sphinx`-friendly --- .../auto/line_profiler.profiler_mixin.rst | 8 +++ docs/source/auto/line_profiler.rst | 3 +- line_profiler/profiler_mixin.py | 60 ++++++++++++------- 3 files changed, 49 insertions(+), 22 deletions(-) create mode 100644 docs/source/auto/line_profiler.profiler_mixin.rst diff --git a/docs/source/auto/line_profiler.profiler_mixin.rst b/docs/source/auto/line_profiler.profiler_mixin.rst new file mode 100644 index 00000000..82e600f1 --- /dev/null +++ b/docs/source/auto/line_profiler.profiler_mixin.rst @@ -0,0 +1,8 @@ +line\_profiler.profiler\_mixin module +===================================== + +.. automodule:: line_profiler.profiler_mixin + :members: + :undoc-members: + :show-inheritance: + :private-members: diff --git a/docs/source/auto/line_profiler.rst b/docs/source/auto/line_profiler.rst index 5e5cba62..a2b108b3 100644 --- a/docs/source/auto/line_profiler.rst +++ b/docs/source/auto/line_profiler.rst @@ -18,9 +18,10 @@ Submodules line_profiler.__main__ line_profiler._line_profiler line_profiler.explicit_profiler - line_profiler.scoping_policy line_profiler.ipython_extension line_profiler.line_profiler + line_profiler.profiler_mixin + line_profiler.scoping_policy Module contents --------------- diff --git a/line_profiler/profiler_mixin.py b/line_profiler/profiler_mixin.py index 0c155bd3..7b6f91db 100644 --- a/line_profiler/profiler_mixin.py +++ b/line_profiler/profiler_mixin.py @@ -61,15 +61,17 @@ def is_cached_property(f): class ByCountProfilerMixin: """ Mixin class for profiler methods built around the - `.enable_by_count()` and `.disable_by_count()` methods, rather than - the `.enable()` and `.disable()` methods. + :py:meth:`!enable_by_count()` and :py:meth:`!disable_by_count()` + methods, rather than the :py:meth:`!enable()` and + :py:meth:`!disable()` methods. - Used by `line_profiler.line_profiler.LineProfiler` and - `kernprof.ContextualProfile`. + Used by :py:class:`line_profiler.line_profiler.LineProfiler` and + :py:class:`kernprof.ContextualProfile`. """ def wrap_callable(self, func): - """ Decorate a function to start the profiler on function entry and stop - it on function exit. + """ + Decorate a function to start the profiler on function entry and + stop it on function exit. """ if is_classmethod(func): return self.wrap_classmethod(func) @@ -173,12 +175,13 @@ def _wrap_callable_wrapper(self, wrapper, impl_attrs, *, Args: wrapper (W): - Wrapper object around regular callables, like - `property`, `staticmethod`, `functools.partial`, etc. + Wrapper object around other callables, like + :py:class:`property`, :py:func:`staticmethod`, + :py:func:`functools.partial`, etc. impl_attrs (Sequence[str]): Attribute names whence to retrieve the individual - callables to be wrapped and profiled, like `.fget`, - `.fset`, and `.fdel` for `property`; + callables to be wrapped and profiled, like ``.fget``, + ``.fset``, and ``.fdel`` for :py:class:`property`; the retrieved values are wrapped and passed as positional arguments to the wrapper constructor. args (Optional[str | Sequence[str]]): @@ -192,16 +195,18 @@ def _wrap_callable_wrapper(self, wrapper, impl_attrs, *, retrieve extra keyword arguments to pass to the wrapper constructor; if a single name, the retrieved values is unpacked; - else, the attribute of `wrapper` at the mapping value is - used to populate the keyword arg at the mapping key. + else, the attribute of ``wrapper`` at the mapping value + is used to populate the keyword arg at the mapping key. name_attr (Optional[str]): Optional attribute name whence to retrieve the name of - `wrapper` to be carried over in the new wrapper, like - `__name__` for `property` (Python 3.13+) and `attrname` - for `functools.cached_property`. + ``wrapper`` to be carried over in the new wrapper, like + ``.__name__`` for :py:class:`property` (Python 3.13+) + and ``.attrname`` for + :py:func:`functools.cached_property`. Returns: - (W): new wrapper of the type of `wrapper` + new_wrapper (W): + New wrapper of the type of ``wrapper`` """ # Wrap implementations impls = [getattr(wrapper, attr) for attr in impl_attrs] @@ -248,7 +253,8 @@ def _wrap_callable_wrapper(self, wrapper, impl_attrs, *, def _wrap_class_and_static_method(self, func): """ - Wrap a class/static method to profile it. + Wrap a :py:func:`classmethod` or :py:func:`staticmethod` to + profile it. """ return self._wrap_callable_wrapper(func, ('__func__',)) @@ -256,14 +262,15 @@ def _wrap_class_and_static_method(self, func): def wrap_boundmethod(self, func): """ - Wrap a bound method to profile it. + Wrap a :py:class:`types.MethodType` to profile it. """ return self._wrap_callable_wrapper(func, ('__func__',), args=('__self__',)) def _wrap_partial(self, func): """ - Wrap a `functools.partial[method]` to profile it. + Wrap a :py:func:`functools.partial` or + :py:class:`functools.partialmethod` to profile it. """ return self._wrap_callable_wrapper(func, ('func',), args='args', kwargs='keywords') @@ -272,7 +279,7 @@ def _wrap_partial(self, func): def wrap_property(self, func): """ - Wrap a property to profile it. + Wrap a :py:class:`property` to profile it. """ return self._wrap_callable_wrapper(func, ('fget', 'fset', 'fdel'), kwargs={'doc': '__doc__'}, @@ -280,7 +287,7 @@ def wrap_property(self, func): def wrap_cached_property(self, func): """ - Wrap a `functools.cached_property` to profile it. + Wrap a :py:func:`functools.cached_property` to profile it. """ return self._wrap_callable_wrapper(func, ('func',), name_attr='attrname') @@ -377,6 +384,17 @@ def wrap_class(self, func): """ Wrap a class by wrapping all locally-defined callables and callable wrappers. + + Returns: + func (type): + The class passed in, with its locally-defined + callables and wrappers wrapped. + + Warns: + UserWarning + If any of the locally-defined callables and wrappers + cannot be replaced with the appropriate wrapper returned + from :py:meth:`.wrap_callable()`. """ get_filter = self._class_scoping_policy.get_filter func_check = get_filter(func, 'func') From b52bf7dcb5033654f55b352ae9adfaf2faa73837 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 27 May 2025 09:19:05 +0200 Subject: [PATCH 41/70] Added tests for class wrapping tests/test_line_profiler.py test_class_decorator() New test for using `LineProfiler` as a class decorator test_add_class_wrapper() New test for when `LineProfiler.add_class()` encounters a callable wrapper which wraps a class instead of a function (See discussion at https://github.com/pyutils/line_profiler/pull/337#issuecomment-2910322662) --- tests/test_line_profiler.py | 80 ++++++++++++++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 1 deletion(-) diff --git a/tests/test_line_profiler.py b/tests/test_line_profiler.py index 1a1e676a..954c77b1 100644 --- a/tests/test_line_profiler.py +++ b/tests/test_line_profiler.py @@ -470,7 +470,7 @@ def foo(self, foo) -> None: def test_cached_property_decorator(): """ - Test for `LineProfiler.wrap_cached_property()` + Test for `LineProfiler.wrap_cached_property()`. Notes ----- @@ -506,6 +506,84 @@ def foo(self) -> int: assert profile.enable_count == 0 +def test_class_decorator(): + """ + Test for `LineProfiler.wrap_class()`. + """ + profile = LineProfiler() + + def unrelated(x): + return str(x) + + @profile + class Object: + def __init__(self, x): + self.x = self.convert(x) + + @property + def id(self): + return id(self) + + @classmethod + def class_method(cls, n): + return cls.__name__ * n + + # This is unrelated to `Object` and shouldn't be profiled + convert = staticmethod(unrelated) + + # Are we keeping tabs on the correct entities? + assert len(profile.functions) == 3 + assert set(profile.functions) == { + Object.__init__.__wrapped__, + Object.id.fget.__wrapped__, + vars(Object)['class_method'].__func__.__wrapped__} + # Make some calls + assert not profile.enable_count + obj = Object(1) + assert obj.x == '1' + assert id(obj) == obj.id + assert obj.class_method(3) == 'ObjectObjectObject' + assert not profile.enable_count + # Check the profiling results + all_entries = sorted(sum(profile.get_stats().timings.values(), [])) + assert len(all_entries) == 3 + assert all(nhits == 1 for (_, nhits, _) in all_entries) + + +def test_add_class_wrapper(): + """ + Test adding a callable-wrapper object wrapping a class. + """ + profile = LineProfiler() + + class Object: + @classmethod + class method: + def __init__(self, cls, x): + self.cls = cls + self.x = x + + def __repr__(self): + fmt = '{.__name__}.{.__name__}({!r})'.format + return fmt(self.cls, type(self), self.x) + + # Bookkeeping + profile.add_class(Object) + method_cls = vars(Object)['method'].__func__ + assert profile.functions == [method_cls.__init__, method_cls.__repr__] + # Actual profiling + with profile: + obj = Object.method(1) + assert obj.cls == Object + assert obj.x == 1 + assert repr(obj) == 'Object.method(1)' + # Check data + all_nhits = { + func_name.rpartition('.')[-1]: sum(nhits for (_, nhits, _) in entries) + for (*_, func_name), entries in profile.get_stats().timings.items()} + assert all_nhits['__init__'] == all_nhits['__repr__'] == 2 + + @pytest.mark.parametrize('decorate', [True, False]) def test_profiler_c_callable_no_op(decorate): """ From 99ab94b932e485ae5efb632c7141695c4dcfe4df Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 28 May 2025 14:52:39 +0200 Subject: [PATCH 42/70] `kernprof`: introduced verbosity levels kernprof.py main() --verbose New flag (`-v` and `--view` now aliases thereto) for increasing verbosity (old `--view` behavior equivalent to level 1; diagnostic output at level 2) -q, --quiet New flag for decreasing verbosity (e.g. suppressing help messages or output of the profiled script) _write_tempfile() Now printing the location of the tempfile as diagnostic output _write_preimports() - Changed call signature - Now printing the generated file as diagnostic output - Now using `rich` (where available and requested) to highlight the generated file --- kernprof.py | 106 ++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 87 insertions(+), 19 deletions(-) diff --git a/kernprof.py b/kernprof.py index 29088eb5..94985fa4 100755 --- a/kernprof.py +++ b/kernprof.py @@ -131,12 +131,15 @@ def main(): import threading import asyncio # NOQA import concurrent.futures # NOQA +import contextlib import tempfile import time import traceback import warnings from argparse import ArgumentError, ArgumentParser +from io import StringIO from runpy import run_module +from shlex import quote # NOTE: This version needs to be manually maintained in # line_profiler/line_profiler.py and line_profiler/__init__.py as well @@ -152,6 +155,9 @@ def main(): from line_profiler.profiler_mixin import ByCountProfilerMixin +DIAGNOSITICS_VERBOSITY = 2 + + def execfile(filename, globals=None, locals=None): """ Python 3.x doesn't have :py:func:`execfile` builtin """ with open(filename, 'rb') as f: @@ -466,8 +472,19 @@ def positive_float(value): "--line-by-line, 'scriptname.prof' without)") parser.add_argument('-s', '--setup', help='Code to execute before the code to profile') - parser.add_argument('-v', '--view', action='store_true', - help='View the results of the profile in addition to saving it') + parser.add_argument('-v', '--verbose', '--view', + action='count', default=0, + help='Increase verbosity level. ' + 'At level 1, view the profiling results ' + 'in addition to saving them; ' + 'at level 2, show other diagnostic info.') + parser.add_argument('-q', '--quiet', + action='count', default=0, + help='Decrease verbosity level. ' + 'At level -1, disable helpful messages ' + '(e.g. "Wrote profile results to <...>"); ' + 'at level -2, silence the stdout; ' + 'at level -3, silence the stderr.') parser.add_argument('-r', '--rich', action='store_true', help='Use rich formatting if viewing output') parser.add_argument('-u', '--unit', default='1e-6', type=positive_float, @@ -514,6 +531,10 @@ def positive_float(value): # Hand off to the dummy parser if necessary to generate the help # text options = real_parser.parse_args(args) + options.verbose -= options.quiet + options.message = functools.partial(_message, options.verbose) + options.diagnostics = functools.partial( + _message, options.verbose, DIAGNOSITICS_VERBOSITY) if help_parser and getattr(options, 'help', False): help_parser.print_help() exit() @@ -533,12 +554,17 @@ def positive_float(value): elif options.script == '-' and not module: tempfile_source_and_content = 'stdin', sys.stdin.read() - if tempfile_source_and_content: - with tempfile.TemporaryDirectory() as tmpdir: + with contextlib.ExitStack() as stack: + enter = stack.enter_context + if options.verbose < -1: # Suppress stdout + devnull = enter(open(os.devnull, mode='w')) + enter(contextlib.redirect_stdout(devnull)) + if options.verbose < -2: # Suppress stderr + enter(contextlib.redirect_stderr(devnull)) + if tempfile_source_and_content: + tmpdir = enter(tempfile.TemporaryDirectory()) _write_tempfile(*tempfile_source_and_content, options, tmpdir) - return _main(options, module) - else: - return _main(options, module) + _main(options, module) def _write_tempfile(source, content, options, tmpdir): @@ -554,6 +580,7 @@ def _write_tempfile(source, content, options, tmpdir): # Do what 3.14 does (#103998)... and also just to be user-friendly content = textwrap.dedent(content) fname = os.path.join(tmpdir, file_prefix + '.py') + options.diagnostics(f'Wrote temporary script file to {fname!r}') with open(fname, mode='w') as fobj: print(content, file=fobj) options.script = fname @@ -571,7 +598,7 @@ def _write_tempfile(source, content, options, tmpdir): suffix='.' + extension) -def _write_preimports(prof, prof_mod, exclude): +def _write_preimports(prof, options, exclude): """ Called by :py:func:`main()` to handle eager pre-imports; not to be invoked on its own. @@ -585,7 +612,7 @@ def _write_preimports(prof, prof_mod, exclude): filtered_targets = [] recurse_targets = [] invalid_targets = [] - for target in prof_mod: + for target in options.prof_mod: if is_dotted_path(target): modname = target else: @@ -633,10 +660,40 @@ def _write_preimports(prof, prof_mod, exclude): if name not in sys.modules) with tempfile.TemporaryDirectory() as tmpdir: temp_mod_path = os.path.join(tmpdir, temp_mod_name + '.py') - with open(temp_mod_path, mode='w') as fobj: - write_eager_import_module(filtered_targets, - recurse=recurse_targets, - stream=fobj) + write_module = functools.partial( + write_eager_import_module, filtered_targets, + recurse=recurse_targets) + temp_file = open(temp_mod_path, mode='w') + if options.verbose >= DIAGNOSITICS_VERBOSITY: + with StringIO() as sio: + write_module(stream=sio) + code = sio.getvalue() + with temp_file as fobj: + print(code, file=fobj) + options.diagnostics('Wrote temporary module for pre-imports ' + f'to {temp_mod_path!r}:\n') + nlines = code.count('\n') + 1 + line_number_width = max(len(str(nlines)), 4) + code_with_lineno = ''.join( + f'{n:>{line_number_width}} {line}' + for n, line in zip(range(1, nlines + 1), + code.splitlines(keepends=True))) + if options.rich: + try: + from rich.console import Console + from rich.syntax import Syntax + except ImportError: + options.diagnostics(code_with_lineno) + else: + options.diagnostics(Syntax(code, 'python', + line_numbers=True), + print=Console().print) + else: + options.diagnostics(code_with_lineno) + options.diagnostics() + else: + with temp_file as fobj: + write_module(stream=fobj) ns = {} # Use a fresh namespace try: execfile(temp_mod_path, ns, ns) @@ -650,6 +707,15 @@ def _write_preimports(prof, prof_mod, exclude): raise +def _message(verbosity, threshold=0, /, *args, print=print, **kwargs): + if isinstance(threshold, str): + args = [threshold, *args] + threshold = 0 + if verbosity < threshold: + return + print(*args, **kwargs) + + def _main(options, module=False): """ Called by :py:func:`main()` for the actual execution and profiling @@ -730,7 +796,7 @@ def _main(options, module=False): # even have a `if __name__ == '__main__': ...` guard. So don't # eager-import it. exclude = set() if module else {script_file} - _write_preimports(prof, options.prof_mod, exclude) + _write_preimports(prof, options, exclude) if options.output_interval: rt = RepeatedTimer(max(options.output_interval, 1), prof.dump_stats, options.outfile) @@ -763,8 +829,8 @@ def _main(options, module=False): if options.output_interval: rt.stop() prof.dump_stats(options.outfile) - print('Wrote profile results to %s' % options.outfile) - if options.view: + options.message(f'Wrote profile results to {options.outfile!r}') + if options.verbose > 0: if isinstance(prof, ContextualProfile): prof.print_stats() else: @@ -773,12 +839,14 @@ def _main(options, module=False): rich=options.rich, stream=original_stdout) else: - print('Inspect results with:') py_exe = _python_command() if isinstance(prof, ContextualProfile): - print(f'{py_exe} -m pstats "{options.outfile}"') + show_mod = 'pstats' else: - print(f'{py_exe} -m line_profiler -rmt "{options.outfile}"') + show_mod = 'line_profiler -rmt' + options.message('Inspect results with:\n' + f'{quote(py_exe)} -m {show_mod} ' + f'{quote(options.outfile)}') # Fully disable the profiler for _ in range(prof.enable_count): prof.disable_by_count() From a1fb5efeef910c4e1644422d3da50de2a76e6c3f Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 28 May 2025 17:16:45 +0200 Subject: [PATCH 43/70] Better diagnostics output kernprof.py::main() --verbose - Now prefixing diagnostic output with '[kernprof ]' - Added diagnostic outputs for: - Implicitly chosen `--outfile` destination - Function call used to run the profiled code --rich - Now unsetting the flag when `rich` cannot be imported - Now using `rich` to highlight all diagnostics if set TODO: tests --- kernprof.py | 153 ++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 107 insertions(+), 46 deletions(-) diff --git a/kernprof.py b/kernprof.py index 94985fa4..d7e514d5 100755 --- a/kernprof.py +++ b/kernprof.py @@ -137,9 +137,12 @@ def main(): import traceback import warnings from argparse import ArgumentError, ArgumentParser +from datetime import datetime from io import StringIO from runpy import run_module from shlex import quote +from textwrap import indent, dedent +from types import MethodType # NOTE: This version needs to be manually maintained in # line_profiler/line_profiler.py and line_profiler/__init__.py as well @@ -424,6 +427,51 @@ def positive_float(value): raise ArgumentError return val + def print_message(threshold=0, /, *args, print=print, **kwargs): + if isinstance(threshold, str): + args = [threshold, *args] + threshold = 0 + if options.verbose < threshold: + return + print(*args, **kwargs) + + def print_diagnostics(*args, **kwargs): + if options.rich: + from rich.console import Console + from rich.markup import escape + + printer = Console().print + else: + escape = str + printer = print + + if args: + now = datetime.now().isoformat(sep=' ', timespec='seconds') + args = ['{} {}'.format(escape(f'[kernprof {now}]'), args[0]), + *args[1:]] + kwargs['print'] = printer + print_message(DIAGNOSITICS_VERBOSITY, *args, **kwargs) + + def print_code_block_diagnostics( + header, code, *, line_numbers=True, **kwargs): + if not header.endswith('\n'): + header += '\n' + if options.rich: + from rich.syntax import Syntax + + code_repr = Syntax(code, 'python', line_numbers=line_numbers) + elif line_numbers: + nlines = code.count('\n') + 1 + line_number_width = max(len(str(nlines)), 4) + code_repr = ''.join( + f'{n:>{line_number_width}} {line}' + for n, line in zip(range(1, nlines + 1), + code.splitlines(keepends=True))) + else: + code_repr = code + kwargs['sep'] = '\n' + print_diagnostics(header, code_repr, **kwargs) + create_parser = functools.partial( ArgumentParser, description='Run and profile a python script.') @@ -531,10 +579,6 @@ def positive_float(value): # Hand off to the dummy parser if necessary to generate the help # text options = real_parser.parse_args(args) - options.verbose -= options.quiet - options.message = functools.partial(_message, options.verbose) - options.diagnostics = functools.partial( - _message, options.verbose, DIAGNOSITICS_VERBOSITY) if help_parser and getattr(options, 'help', False): help_parser.print_help() exit() @@ -554,6 +598,20 @@ def positive_float(value): elif options.script == '-' and not module: tempfile_source_and_content = 'stdin', sys.stdin.read() + # Handle output + options.verbose -= options.quiet + options.message = (print_message + if options.verbose < DIAGNOSITICS_VERBOSITY else + print_diagnostics) + options.diagnostics = print_diagnostics + options.code_diagnostics = print_code_block_diagnostics + if options.rich: + try: + import rich # noqa: F401 + except ImportError: + options.rich = False + options.diagnostics('`rich` not installed, unsetting --rich') + with contextlib.ExitStack() as stack: enter = stack.enter_context if options.verbose < -1: # Suppress stdout @@ -573,12 +631,10 @@ def _write_tempfile(source, content, options, tmpdir): :command:`kernprof -`; not to be invoked on its own. """ - import textwrap - # Set up the script to be run file_prefix = f'kernprof-{source}' # Do what 3.14 does (#103998)... and also just to be user-friendly - content = textwrap.dedent(content) + content = dedent(content) fname = os.path.join(tmpdir, file_prefix + '.py') options.diagnostics(f'Wrote temporary script file to {fname!r}') with open(fname, mode='w') as fobj: @@ -596,6 +652,8 @@ def _write_tempfile(source, content, options, tmpdir): _, options.outfile = tempfile.mkstemp(dir=os.curdir, prefix=file_prefix + '-', suffix='.' + extension) + options.diagnostics( + f'Using default output destination {options.outfile!r}') def _write_preimports(prof, options, exclude): @@ -670,27 +728,10 @@ def _write_preimports(prof, options, exclude): code = sio.getvalue() with temp_file as fobj: print(code, file=fobj) - options.diagnostics('Wrote temporary module for pre-imports ' - f'to {temp_mod_path!r}:\n') - nlines = code.count('\n') + 1 - line_number_width = max(len(str(nlines)), 4) - code_with_lineno = ''.join( - f'{n:>{line_number_width}} {line}' - for n, line in zip(range(1, nlines + 1), - code.splitlines(keepends=True))) - if options.rich: - try: - from rich.console import Console - from rich.syntax import Syntax - except ImportError: - options.diagnostics(code_with_lineno) - else: - options.diagnostics(Syntax(code, 'python', - line_numbers=True), - print=Console().print) - else: - options.diagnostics(code_with_lineno) - options.diagnostics() + options.code_diagnostics( + 'Wrote temporary module for pre-imports ' + f'to {temp_mod_path!r}:', + code) else: with temp_file as fobj: write_module(stream=fobj) @@ -707,24 +748,36 @@ def _write_preimports(prof, options, exclude): raise -def _message(verbosity, threshold=0, /, *args, print=print, **kwargs): - if isinstance(threshold, str): - args = [threshold, *args] - threshold = 0 - if verbosity < threshold: - return - print(*args, **kwargs) - - def _main(options, module=False): """ Called by :py:func:`main()` for the actual execution and profiling of code; not to be invoked on its own. """ + def call_with_diagnostics(func, *args, **kwargs): + import reprlib + + if options.verbose < DIAGNOSITICS_VERBOSITY: + return func(*args, **kwargs) + if isinstance(func, MethodType): + obj = func.__self__ + func_repr = ('{0.__module__}.{0.__qualname__}(...).{1.__name__}' + .format(type(obj), func.__func__)) + else: + func_repr = '{0.__module__}.{0.__qualname__}'.format(func) + get_repr = reprlib.repr + args_reprs = ',\n'.join( + [get_repr(arg) for arg in args] + + [f'{arg}={get_repr(value)}' for arg, value in kwargs.items()]) + call = '{}(\n{})\n'.format(func_repr, indent(args_reprs, ' ')) + options.code_diagnostics('Calling:', call) + return func(*args, **kwargs) + if not options.outfile: extension = 'lprof' if options.line_by_line else 'prof' - options.outfile = '%s.%s' % (os.path.basename(options.script), extension) + options.outfile = f'{os.path.basename(options.script)}.{extension}' + options.diagnostics( + f'Using default output destination {options.outfile!r}') sys.argv = [options.script] + options.args if module: @@ -743,6 +796,8 @@ def _main(options, module=False): # kernprof.py's. sys.path.insert(0, os.path.dirname(setup_file)) ns = locals() + options.diagnostics( + f'Executing file {setup_file!r} as pre-profiling setup') execfile(setup_file, ns, ns) if options.line_by_line: @@ -811,18 +866,24 @@ def _main(options, module=False): ns = locals() if options.prof_mod and options.line_by_line: from line_profiler.autoprofile import autoprofile - autoprofile.run(script_file, ns, - prof_mod=options.prof_mod, - profile_imports=options.prof_imports, - as_module=module is not None) + + call_with_diagnostics( + autoprofile.run, script_file, ns, + prof_mod=options.prof_mod, + profile_imports=options.prof_imports, + as_module=module is not None) elif module and options.builtin: - rmod_(options.script, ns) + call_with_diagnostics(rmod_, options.script, ns) elif options.builtin: - execfile(script_file, ns, ns) + call_with_diagnostics(execfile, script_file, ns, ns) elif module: - prof.runctx(f'rmod_({options.script!r}, globals())', ns, ns) + call_with_diagnostics( + prof.runctx, f'rmod_({options.script!r}, globals())', + ns, ns) else: - prof.runctx('execfile_(%r, globals())' % (script_file,), ns, ns) + call_with_diagnostics( + prof.runctx, f'execfile_({script_file!r}, globals())', + ns, ns) except (KeyboardInterrupt, SystemExit): pass finally: From 43bf3622a3459e02a9a3550d5cb2d1df88ab5901 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 28 May 2025 18:07:42 +0200 Subject: [PATCH 44/70] More refactoring kernprof.py main() - Made the formatting of code more consistent between `--rich` and non-rich mode - Added more diagnostic output regarding the script run and the parsed arguments _write_tempfile() Added (syntax-highlighted) diagnostic output for the tempfile _main() - Refactored internal function used for formatting function calls, using `pprint.pformat()` to indent the function-call arguments - Replaced use of `locals()` with explicitly-constructed namespaces --- kernprof.py | 60 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 20 deletions(-) diff --git a/kernprof.py b/kernprof.py index d7e514d5..695fb021 100755 --- a/kernprof.py +++ b/kernprof.py @@ -140,9 +140,10 @@ def main(): from datetime import datetime from io import StringIO from runpy import run_module +from pprint import pformat from shlex import quote from textwrap import indent, dedent -from types import MethodType +from types import MethodType, SimpleNamespace # NOTE: This version needs to be manually maintained in # line_profiler/line_profiler.py and line_profiler/__init__.py as well @@ -454,15 +455,13 @@ def print_diagnostics(*args, **kwargs): def print_code_block_diagnostics( header, code, *, line_numbers=True, **kwargs): - if not header.endswith('\n'): - header += '\n' if options.rich: from rich.syntax import Syntax code_repr = Syntax(code, 'python', line_numbers=line_numbers) elif line_numbers: nlines = code.count('\n') + 1 - line_number_width = max(len(str(nlines)), 4) + line_number_width = len(str(nlines)) + 2 code_repr = ''.join( f'{n:>{line_number_width}} {line}' for n, line in zip(range(1, nlines + 1), @@ -470,7 +469,14 @@ def print_code_block_diagnostics( else: code_repr = code kwargs['sep'] = '\n' - print_diagnostics(header, code_repr, **kwargs) + + # Insert additional space + if not header.endswith('\n'): + header += '\n' + args = [header, code_repr] + if not code.endswith('\n'): + args.append('') + print_diagnostics(*args, **kwargs) create_parser = functools.partial( ArgumentParser, @@ -578,7 +584,7 @@ def print_code_block_diagnostics( # Hand off to the dummy parser if necessary to generate the help # text - options = real_parser.parse_args(args) + options = SimpleNamespace(**vars(real_parser.parse_args(args))) if help_parser and getattr(options, 'help', False): help_parser.print_help() exit() @@ -612,6 +618,15 @@ def print_code_block_diagnostics( options.rich = False options.diagnostics('`rich` not installed, unsetting --rich') + options.code_diagnostics('Parser output:', pformat(options)) + if module is not None: + options.diagnostics('Profiling module:', module) + elif tempfile_source_and_content: + options.diagnostics('Profiling script read from', + tempfile_source_and_content[0]) + else: + options.diagnostics('Profiling script:', options.script) + with contextlib.ExitStack() as stack: enter = stack.enter_context if options.verbose < -1: # Suppress stdout @@ -636,9 +651,10 @@ def _write_tempfile(source, content, options, tmpdir): # Do what 3.14 does (#103998)... and also just to be user-friendly content = dedent(content) fname = os.path.join(tmpdir, file_prefix + '.py') - options.diagnostics(f'Wrote temporary script file to {fname!r}') with open(fname, mode='w') as fobj: print(content, file=fobj) + options.code_diagnostics(f'Wrote temporary script file to {fname!r}:', + content) options.script = fname # Add the tempfile to `--prof-mod` if options.prof_mod: @@ -755,8 +771,6 @@ def _main(options, module=False): not to be invoked on its own. """ def call_with_diagnostics(func, *args, **kwargs): - import reprlib - if options.verbose < DIAGNOSITICS_VERBOSITY: return func(*args, **kwargs) if isinstance(func, MethodType): @@ -765,11 +779,19 @@ def call_with_diagnostics(func, *args, **kwargs): .format(type(obj), func.__func__)) else: func_repr = '{0.__module__}.{0.__qualname__}'.format(func) - get_repr = reprlib.repr - args_reprs = ',\n'.join( - [get_repr(arg) for arg in args] - + [f'{arg}={get_repr(value)}' for arg, value in kwargs.items()]) - call = '{}(\n{})\n'.format(func_repr, indent(args_reprs, ' ')) + args_repr = dedent(' ' + pformat(args)[len('['):-len(']')]) + kwargs_repr = dedent( + ' ' * len('namespace(') + + pformat(SimpleNamespace(**kwargs))[len('namespace('):-len(')')]) + if args_repr and kwargs_repr: + all_args_repr = f'{args_repr},\n{kwargs_repr}' + else: + all_args_repr = args_repr or kwargs_repr + if all_args_repr: + call = '{}(\n{})'.format( + func_repr, indent(all_args_repr, ' ')) + else: + call = func_repr + '()' options.code_diagnostics('Calling:', call) return func(*args, **kwargs) @@ -790,12 +812,10 @@ def call_with_diagnostics(func, *args, **kwargs): # Run some setup code outside of the profiler. This is good for large # imports. setup_file = find_script(options.setup) - __file__ = setup_file - __name__ = '__main__' # Make sure the script's directory is on sys.path instead of just # kernprof.py's. sys.path.insert(0, os.path.dirname(setup_file)) - ns = locals() + ns = {'__file__': setup_file, '__name__': '__main__'} options.diagnostics( f'Executing file {setup_file!r} as pre-profiling setup') execfile(setup_file, ns, ns) @@ -832,8 +852,6 @@ def call_with_diagnostics(func, *args, **kwargs): # Make sure the script's directory is on sys.path instead of # just kernprof.py's. sys.path.insert(0, os.path.dirname(script_file)) - __file__ = script_file - __name__ = '__main__' # If using eager pre-imports, write a dummy module which contains # all those imports and marks them for profiling, then run it @@ -863,7 +881,9 @@ def call_with_diagnostics(func, *args, **kwargs): execfile_ = execfile rmod_ = functools.partial(run_module, run_name='__main__', alter_sys=True) - ns = locals() + ns = {'__file__': script_file, '__name__': '__main__', + 'execfile_': execfile_, 'rmod_': rmod_, + 'prof': prof} if options.prof_mod and options.line_by_line: from line_profiler.autoprofile import autoprofile From 2153b188492b5c9d479a001d00a9c8fd8b9c61eb Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Thu, 29 May 2025 00:04:33 +0200 Subject: [PATCH 45/70] Test and doc fixes kernprof.py::__doc__ Updated `kernprof --help` output tests/test_kernprof.py::test_kernprof_verbosity() New test for verbosity and output --- kernprof.py | 8 +++- tests/test_kernprof.py | 84 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 2 deletions(-) diff --git a/kernprof.py b/kernprof.py index 695fb021..76759634 100755 --- a/kernprof.py +++ b/kernprof.py @@ -73,7 +73,8 @@ def main(): .. code:: - usage: kernprof [-h] [-V] [-l] [-b] [-o OUTFILE] [-s SETUP] [-v] [-r] [-u UNIT] [-z] [-i [OUTPUT_INTERVAL]] [-p PROF_MOD] [--prof-imports] + usage: kernprof [-h] [-V] [-l] [-b] [-o OUTFILE] [-s SETUP] [-v] [-q] [-r] [-u UNIT] [-z] [-i [OUTPUT_INTERVAL]] [-p {path/to/script | object.dotted.path}[,...]] + [--no-preimports] [--prof-imports] {path/to/script | -m path.to.module | -c "literal code"} ... Run and profile a python script. @@ -91,7 +92,10 @@ def main(): -o, --outfile OUTFILE Save stats to (default: 'scriptname.lprof' with --line-by-line, 'scriptname.prof' without) -s, --setup SETUP Code to execute before the code to profile - -v, --view View the results of the profile in addition to saving it + -v, --verbose, --view + Increase verbosity level. At level 1, view the profiling results in addition to saving them; at level 2, show other diagnostic info. + -q, --quiet Decrease verbosity level. At level -1, disable helpful messages (e.g. "Wrote profile results to <...>"); at level -2, silence the stdout; at level -3, + silence the stderr. -r, --rich Use rich formatting if viewing output -u, --unit UNIT Output unit (in seconds) in which the timing info is displayed (default: 1e-6) -z, --skip-zero Hide functions which have not been called diff --git a/tests/test_kernprof.py b/tests/test_kernprof.py index c56dd215..cf102b27 100644 --- a/tests/test_kernprof.py +++ b/tests/test_kernprof.py @@ -177,6 +177,90 @@ def main(): assert sys.modules.get('__main__') is old_main +@pytest.mark.parametrize( + ('flags', 'expected_stdout', 'expected_stderr'), + [('', # Neutral verbosity level + {'^Output to stdout': True, + r"^Wrote .* '.*script\.py\.lprof'": True, + r'Parser output:''(?:\n)+'r'.*namespace\((?:.+,\n)*.*\)': False, + r'^Inspect results with:''\n' + r'python -m line_profiler .*script\.py\.lprof': True, + '^ *[0-9]+ *import zipfile': False, + r'line_profiler\.autoprofile\.autoprofile' + r'\.run\(\n(?:.+,\n)*.*\)': False, + r'^\[kernprof .*\]': False, + r'^Function: main': False}, + {'^Output to stderr': True}), + ('--view', # Verbosity level 1 = `--view` + {'^Output to stdout': True, + r"^Wrote .* '.*script\.py\.lprof'": True, + r'Parser output:''(?:\n)+'r'.*namespace\((?:.+,\n)*.*\)': False, + r'^Inspect results with:''\n' + r'python -m line_profiler .*script\.py\.lprof': False, + '^ *[0-9]+ *import zipfile': False, + r'line_profiler\.autoprofile\.autoprofile' + r'\.run\(\n(?:.+,\n)*.*\)': False, + r'^\[kernprof .*\]': False, + r'^Function: main': True}, + {'^Output to stderr': True}), + ('-vv', # Verbosity level 2, show diagnostics + {'^Output to stdout': True, + r"^\[kernprof .*\] Wrote .* '.*script\.py\.lprof'": True, + r'Parser output:''(?:\n)+'r'.*namespace\((?:.+,\n)*.*\)': True, + '^ *[0-9]+ *import zipfile': True, + r'Inspect results with:''\n' + r'python -m line_profiler .*script\.py\.lprof': False, + r'line_profiler\.autoprofile\.autoprofile' + r'\.run\(\n(?:.+,\n)*.*\)': True, + r'^Function: main': True}, + {'^Output to stderr': True}), + # Verbosity level -1, suppress `kernprof` output + ('--quiet', + {'^Output to stdout': True, 'Wrote': False}, + {'^Output to stderr': True}), + # Verbosity level -2, suppress script stdout + # (also test verbosity arithmatics) + ('--quiet --quiet --verbose -q', None, {'^Output to stderr': True}), + # Verbosity level -3, suppress script stderr + ('-qq --quiet', None, None)]) +def test_kernprof_verbosity(flags, expected_stdout, expected_stderr): + """ + Test the various verbosity levels of `kernprof`. + """ + with contextlib.ExitStack() as stack: + enter = stack.enter_context + tmpdir = enter(tempfile.TemporaryDirectory()) + temp_dpath = ub.Path(tmpdir) + (temp_dpath / 'script.py').write_text(ub.codeblock( + ''' + import sys + + + def main(): + print('Output to stdout', file=sys.stdout) + print('Output to stderr', file=sys.stderr) + + + if __name__ == '__main__': + main() + ''')) + enter(ub.ChDir(tmpdir)) + proc = ub.cmd(['kernprof', '-l', + # Add an eager pre-import target + '-pscript.py', '-pzipfile', '-z', + *shlex.split(flags), 'script.py']) + proc.check_returncode() + print(proc.stdout) + for expected_outputs, stream in [(expected_stdout, proc.stdout), + (expected_stderr, proc.stderr)]: + if expected_outputs is None: + assert not stream + continue + for pattern, expect_match in expected_outputs.items(): + assert bool(re.search(pattern, stream, + flags=re.MULTILINE)) == expect_match + + class TestKernprof(unittest.TestCase): def test_enable_disable(self): From 7cd55f6467ece412705a3ad9365a510536647416 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Thu, 29 May 2025 00:31:12 +0200 Subject: [PATCH 46/70] More lenient pre-imports line_profiler/autoprofile/eager_preimports.py write_eager_import_module() - Added "# noqa: F821" to the module line defining `adder` - Added one extra blank line between preambles and imports (PEP8) - Fixed bug where errors importing a module are not shown in the warning listing the failed profiling targets - Now distinguishing between explictly provided and implicitly included (via `recurse`) profiling targets, allowing arbitrary errors (instead of just `ImportError`) when importing the latter --- line_profiler/autoprofile/eager_preimports.py | 50 +++++++++++++------ 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/line_profiler/autoprofile/eager_preimports.py b/line_profiler/autoprofile/eager_preimports.py index a10088ce..333ba9fb 100644 --- a/line_profiler/autoprofile/eager_preimports.py +++ b/line_profiler/autoprofile/eager_preimports.py @@ -292,9 +292,10 @@ def write_eager_import_module(dotted_paths, stream=None, *, ... written = strip(sio.getvalue()) ... >>> assert written == strip(''' - ... add = profile.add_imported_function_or_module + ... add = profile.add_imported_function_or_module # noqa: F821 ... failures = [] ... + ... ... try: ... import importlib.abc as module ... except ImportError: @@ -312,7 +313,7 @@ def write_eager_import_module(dotted_paths, stream=None, *, ... try: ... import importlib.util as module ... except ImportError: - ... pass + ... failures.append('importlib.util') ... else: ... add(module) ... @@ -382,6 +383,7 @@ def write_eager_import_module(dotted_paths, stream=None, *, else: recurse = dotted_paths if recurse else set() dotted_paths |= recurse + paths_added_by_recursion = set() imports = {} unknown_locs = [] @@ -399,9 +401,12 @@ def write_eager_import_module(dotted_paths, stream=None, *, recurse_root = None imports.setdefault(module, set()).add(target) # FIXME: how do we handle namespace packages? - if recurse_root is not None: - for info in walk_packages([recurse_root], prefix=module + '.'): - imports.setdefault(info.name, set()).add(None) + if recurse_root is None: + continue + for info in walk_packages([recurse_root], prefix=module + '.'): + imports.setdefault(info.name, set()).add(None) + paths_added_by_recursion.add(info.name) + paths_added_by_recursion -= dotted_paths # Warn against failed imports if unknown_locs: @@ -413,23 +418,40 @@ def write_eager_import_module(dotted_paths, stream=None, *, # Do the imports and add them with `adder` write = functools.partial(print, file=stream) - write(f'{adder_name} = {adder}\n{failures_name} = []') - for module, targets in imports.items(): + write(f'{adder_name} = {adder} # noqa: F821\n{failures_name} = []') + for i, (module, targets) in enumerate(imports.items()): assert targets + # Write one more empty line so that the imports are separated + # from the preambles by 2 lines + if not i: + write() + # Allow arbitrary errors for modules that are only added + # indirectly (via descent/recursion) + if module in paths_added_by_recursion: + allowed_error = 'Exception' + else: + allowed_error = 'ImportError' + # Is the module itself a direct target? + try: + targets.remove(None) + except KeyError: # Not found + profile_whole_module = False + else: + profile_whole_module = True + if profile_whole_module: + on_error = f'{failures_name}.append({module!r})' + else: + on_error = 'pass' write('\n' + strip(f""" try: {indent}import {module} as {module_name} - except ImportError: - {indent}pass + except {allowed_error}: + {indent}{on_error} else: """)) chunks = [] - try: - targets.remove(None) - except KeyError: # Not found - pass - else: # Add the whole module + if profile_whole_module: chunks.append(f'{adder_name}({module_name})') for target in sorted(targets): path = f'{module}.{target}' From c3221edd4d3659a1ad6999092128ae2233b0a8db Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Thu, 29 May 2025 02:15:21 +0200 Subject: [PATCH 47/70] `line_profiler.autoprofile.eager_preimports` tests tests/test_eager_preimports.py test_write_eager_import_module_wrong_adder() - Removed unused parameter `msg` - Added type annotations for the parameters test_written_module_pep8_compliance() New test that the generated module passes linting test_written_module_error_handling() New test that failures in retrieving the profiling targets are appropriately converted into errors and warnings --- tests/test_eager_preimports.py | 163 +++++++++++++++++++++++++++++++-- 1 file changed, 155 insertions(+), 8 deletions(-) diff --git a/tests/test_eager_preimports.py b/tests/test_eager_preimports.py index 00a2b7da..681745af 100644 --- a/tests/test_eager_preimports.py +++ b/tests/test_eager_preimports.py @@ -5,20 +5,167 @@ ----- Most of the features are already covered by the doctests. """ +import subprocess +import sys +from contextlib import ExitStack +from pathlib import Path +from operator import methodcaller +from runpy import run_path +from tempfile import TemporaryDirectory +from textwrap import dedent +from types import SimpleNamespace +from typing import Collection, Generator, Sequence, Type, Optional, Union +from uuid import uuid4 +from warnings import catch_warnings + import pytest +try: + import flake8 # noqa: F401 +except ImportError: + HAS_FLAKE8 = False +else: + HAS_FLAKE8 = True + +from line_profiler.autoprofile.eager_preimports import ( + write_eager_import_module as write_module) + -from line_profiler.autoprofile import eager_preimports +def write(path: Path, content: Optional[str] = None) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + if content is None: + path.touch() + else: + path.write_text(dedent(content).strip('\n')) + + +@pytest.fixture +def sample_module(tmp_path: Path) -> str: + """ + Write a module and put it in :py:data:`sys.path`. When we're done, + reset :py:data:`sys.path` and `sys.modules`. + """ + def gen_names() -> Generator[str, None, None]: + while True: + yield 'my_module_' + str(uuid4()).replace('-', '_') + + old_path = sys.path.copy() + old_modules = sys.modules.copy() + module_name = next(name for name in gen_names() if name not in old_modules) + new_path = tmp_path / '_modules' + write(new_path / module_name / '__init__.py') + write(new_path / module_name / 'foo' / '__init__.py') + write(new_path / module_name / 'foo' / 'bar.py') + write(new_path / module_name / 'foo' / 'baz.py', + """ + ''' + This is a bad module. + ''' + raise AssertionError + """) + write(new_path / module_name / 'foobar.py') + try: + sys.path.insert(0, str(new_path)) + yield module_name + finally: + sys.path.clear() + sys.path[:] = old_path + assert str(new_path) not in sys.path + sys.modules.clear() + sys.modules.update(old_modules) + assert module_name not in sys.modules @pytest.mark.parametrize( - 'adder, xc, msg', - [('foo; bar', ValueError, None), - (1, TypeError, None), - ('(foo\n .bar)', ValueError, None)]) -def test_write_eager_import_module_wrong_adder(adder, xc, msg) -> None: + ('adder', 'xc'), + [('foo; bar', ValueError), (1, TypeError), ('(foo\n .bar)', ValueError)]) +def test_write_eager_import_module_wrong_adder( + adder: str, xc: Type[Exception]) -> None: """ Test passing an erroneous ``adder`` to :py:meth:`~.write_eager_import_module()`. """ - with pytest.raises(xc, match=msg): - eager_preimports.write_eager_import_module(['foo'], adder=adder) + with pytest.raises(xc): + write_module(['foo'], adder=adder) + + +@pytest.mark.skipif(not HAS_FLAKE8, reason='no `flake8`') +def test_written_module_pep8_compliance(sample_module: str): + """ + Test that the module written by + :py:meth:`~.write_eager_import_module()` passes linting by + :py:mod:`flake8`. + """ + with TemporaryDirectory() as tmpdir: + module = Path(tmpdir) / 'module.py' + with module.open(mode='w') as fobj: + write_module([sample_module + '.foobar'], + recurse=[sample_module + '.foo'], stream=fobj) + print(module.read_text()) + (subprocess + .run([sys.executable, '-m', 'flake8', + '--extend-ignore=E501', # Allow long lines + module]) + .check_returncode()) + + +@pytest.mark.parametrize( + ('dotted_paths', 'recurse', 'warnings', 'error'), + [(['__MODULE__.foobar'], ['__MODULE__.foo'], + # `foo.baz` is indirectly included, so its raising an error + # shouldn't cause the script to error out + [{'target cannot', '__MODULE__.foo.baz'}], + None), + # We don't recurse down `__MODULE__.foo`, so that doesn't give a + # warning; but `__MODULE__.baz` cannot be imported because it + # doesn't exist + (['__MODULE__.foo', '__MODULE__.baz'], False, + [{'target cannot', '__MODULE__.baz'}], None), + # If we do recurse however, `__MODULE__.foo.baz` also ends up in + # the warning + # (also there's a `__MODULE___foo` which doesn't exist, about which + # the warning is issued during module generation) + (['__MODULE__' + '_foo', '__MODULE__', '__MODULE__.baz'], True, + [{'target cannot', '__MODULE__' + '_foo'}, # Fails at write + {'targets cannot', # Fails at import + '__MODULE__.foo.baz', '__MODULE__.baz'}], + None), + # And if the problematic module is an explicit target, raise the + # error + (['__MODULE__', '__MODULE__.foo.baz'], False, [], AssertionError)]) +def test_written_module_error_handling( + sample_module: str, + dotted_paths: Collection[str], + recurse: Union[Collection[str], bool], + warnings: Sequence[Collection[str]], + error: Union[Type[Exception], None]): + """ + Test that the module written by + :py:meth:`~.write_eager_import_module()` gracefully handles errors + for implicitly included modules. + """ + replace = methodcaller('replace', '__MODULE__', sample_module) + dotted_paths = [replace(target) for target in dotted_paths] + if recurse not in (True, False): + recurse = [replace(target) for target in recurse] + warnings = [{replace(fragment) for fragment in fragments} + for fragments in warnings] + with TemporaryDirectory() as tmpdir: + module = Path(tmpdir) / 'module.py' + with ExitStack() as stack: + enter = stack.enter_context + # Set up the warning capturing early so that we catch both + # warnings at module-generation time and execution time + captured_warnings = enter(catch_warnings(record=True)) + with module.open(mode='w') as fobj: + write_module(dotted_paths, recurse=recurse, stream=fobj) + print(module.read_text()) + if error is not None: + enter(pytest.raises(error)) + # Just use a dummy object, no need to instantiate a profiler + prof = SimpleNamespace( + add_imported_function_or_module=lambda *_, **__: 0) + run_path(str(module), {'profile': prof}, 'module') + assert len(captured_warnings) == len(warnings) + for warning, fragments in zip(captured_warnings, warnings): + for fragment in fragments: + assert fragment in str(warning.message) From 6bdb9bdd81cbbb9d42d245185754f75863ac5390 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Thu, 29 May 2025 02:58:26 +0200 Subject: [PATCH 48/70] CI fix and CHANGELOG updates CHANGELOG.rst Updated entry tests/test_kernprof.py Loosened problematic subtest which trigger a `CoverageWarning` in CI and fails (because it didn't expect any `stderr` output) --- CHANGELOG.rst | 10 +++++++++- tests/test_kernprof.py | 5 ++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6b96a08e..c6e5085d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -15,7 +15,15 @@ Changes * ENH: In Python >=3.11, profiled objects are reported using their qualified name. * ENH: Highlight final summary using rich if enabled * ENH: Made it possible to use multiple profiler instances simultaneously -* ENH: ``kernprof --prof-mod`` target entities are now imported and profiled regardless of whether they are directly imported in the run script/module/code (old behavior recoed by passing ``--no-preimports``); made on-import profiling more aggressive so that it doesn't miss entities like class methods and properties +* ENH: various improvements related to auto-profiling: + * ``kernprof -p`` target entities are now imported and profiled regardless of + whether they are directly imported in the run script/module/code (old + behavior restored by passing ``--no-preimports``) + * ``kernprof -v`` and the new ``-q`` now control the verbosity level instead + of being a boolean, allowing diagnostic outputs or output suppression + * On-import profiling is now more aggressive so that it doesn't miss entities + like class methods and properties + * ``LineProfiler`` can now be used as a class decorator 4.2.0 ~~~~~ diff --git a/tests/test_kernprof.py b/tests/test_kernprof.py index cf102b27..eed6f576 100644 --- a/tests/test_kernprof.py +++ b/tests/test_kernprof.py @@ -222,7 +222,10 @@ def main(): # (also test verbosity arithmatics) ('--quiet --quiet --verbose -q', None, {'^Output to stderr': True}), # Verbosity level -3, suppress script stderr - ('-qq --quiet', None, None)]) + ('-qq --quiet', None, + # This should have been `None`, but there's something weird with + # `coverage` in CI which causes a spurious warning... + {'^Output to stderr': False})]) def test_kernprof_verbosity(flags, expected_stdout, expected_stderr): """ Test the various verbosity levels of `kernprof`. From c76ab78860418b306cac4bf9de42600afe942416 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 1 Jun 2025 00:04:19 +0200 Subject: [PATCH 49/70] Now skipping Cython callables line_profiler/profiler_mixin.py[i] C_LEVEL_CALLABLE_TYPES Added type for Cython callables is_c_level_callable() Now also returns true for Cython callables tests/test_line_profiler.py::test_add_class_wrapper() Added test case for Cython callables --- line_profiler/profiler_mixin.py | 12 ++-- line_profiler/profiler_mixin.pyi | 106 +++++++++++++++++++++++++++++-- tests/test_line_profiler.py | 7 +- 3 files changed, 112 insertions(+), 13 deletions(-) diff --git a/line_profiler/profiler_mixin.py b/line_profiler/profiler_mixin.py index 7b6f91db..ef803fb5 100644 --- a/line_profiler/profiler_mixin.py +++ b/line_profiler/profiler_mixin.py @@ -2,6 +2,7 @@ import inspect import types from warnings import warn +from ._line_profiler import label from .scoping_policy import ScopingPolicy @@ -10,22 +11,23 @@ is_generator = inspect.isgeneratorfunction is_async_generator = inspect.isasyncgenfunction -# These objects are callables, but are defined in C so we can't handle -# them anyway +# These objects are callables, but are defined in C(-ython) so we can't +# handle them anyway C_LEVEL_CALLABLE_TYPES = (types.BuiltinFunctionType, types.BuiltinMethodType, types.ClassMethodDescriptorType, types.MethodDescriptorType, types.MethodWrapperType, - types.WrapperDescriptorType) + types.WrapperDescriptorType, + type(label)) def is_c_level_callable(func): """ Returns: func_is_c_level (bool): - Whether a callable is defined at the C level (and is thus - non-profilable). + Whether a callable is defined at the C(-ython) level (and is + thus non-profilable). """ return isinstance(func, C_LEVEL_CALLABLE_TYPES) diff --git a/line_profiler/profiler_mixin.pyi b/line_profiler/profiler_mixin.pyi index bce5453a..10c91f7e 100644 --- a/line_profiler/profiler_mixin.pyi +++ b/line_profiler/profiler_mixin.pyi @@ -1,9 +1,10 @@ from functools import cached_property, partial, partialmethod -from types import (FunctionType, MethodType, +from types import (CodeType, FunctionType, MethodType, BuiltinFunctionType, BuiltinMethodType, ClassMethodDescriptorType, MethodDescriptorType, MethodWrapperType, WrapperDescriptorType) -from typing import Any, Callable, Dict, List, Mapping, TypeVar +from typing import (TYPE_CHECKING, + Any, Callable, Dict, List, Mapping, Protocol, TypeVar) try: from typing import ( # type: ignore[attr-defined] # noqa: F401 ParamSpec) @@ -19,16 +20,109 @@ try: TypeIs) except ImportError: # Python < 3.13 from typing_extensions import TypeIs # noqa: F401 +from ._line_profiler import label -CLevelCallable = TypeVar('CLevelCallable', - BuiltinFunctionType, BuiltinMethodType, - ClassMethodDescriptorType, MethodDescriptorType, - MethodWrapperType, WrapperDescriptorType) T = TypeVar('T', bound=type) +T_co = TypeVar('T_co', covariant=True) R = TypeVar('R') PS = ParamSpec('PS') +if TYPE_CHECKING: + class CythonCallable(Protocol[PS, T_co]): + def __call__(self, *args: PS.args, **kwargs: PS.kwargs) -> T_co: + ... + + @property + def __code__(self) -> CodeType: + ... + + @property + def func_code(self) -> CodeType: + ... + + @property + def __name__(self) -> str: + ... + + @property + def func_name(self) -> str: + ... + + @property + def __qualname__(self) -> str: + ... + + @property + def __doc__(self) -> str | None: + ... + + @__doc__.setter + def __doc__(self, doc: str | None) -> None: + ... + + @property + def func_doc(self) -> str | None: + ... + + @property + def __globals__(self) -> Dict[str, Any]: + ... + + @property + def func_globals(self) -> Dict[str, Any]: + ... + + @property + def __dict__(self) -> Dict[str, Any]: + ... + + @__dict__.setter + def __dict__(self, dict: str | None) -> None: + ... + + @property + def func_dict(self) -> Dict[str, Any]: + ... + + @property + def __annotations__(self) -> Dict[str, Any]: + ... + + @__annotations__.setter + def __annotations__(self, annotations: str | None) -> None: + ... + + @property + def __defaults__(self): + ... + + @property + def func_defaults(self): + ... + + @property + def __kwdefaults__(self): + ... + + @property + def __closure__(self): + ... + + @property + def func_closure(self): + ... + + +else: + CythonCallable = type(label) + +CLevelCallable = TypeVar('CLevelCallable', + BuiltinFunctionType, BuiltinMethodType, + ClassMethodDescriptorType, MethodDescriptorType, + MethodWrapperType, WrapperDescriptorType, + CythonCallable) + def is_c_level_callable(func: Any) -> TypeIs[CLevelCallable]: ... diff --git a/tests/test_line_profiler.py b/tests/test_line_profiler.py index 954c77b1..e7df7b21 100644 --- a/tests/test_line_profiler.py +++ b/tests/test_line_profiler.py @@ -587,10 +587,12 @@ def __repr__(self): @pytest.mark.parametrize('decorate', [True, False]) def test_profiler_c_callable_no_op(decorate): """ - Test that the following are no-ops on C-level callables: + Test that the following are no-ops on C(-ython)-level callables: - Decoration (`.__call__()`): the callable is returned as-is. - `.add_callable()`: it returns 0. """ + CythonCallable = type(LineProfiler.enable) + assert not isinstance(LineProfiler.enable, types.FunctionType) profile = LineProfiler() for (func, Type) in [ @@ -599,7 +601,8 @@ def test_profiler_c_callable_no_op(decorate): (vars(int)['from_bytes'], types.ClassMethodDescriptorType), (str.split, types.MethodDescriptorType), ((1).__str__, types.MethodWrapperType), - (int.__repr__, types.WrapperDescriptorType)]: + (int.__repr__, types.WrapperDescriptorType), + (LineProfiler.enable, CythonCallable)]: assert isinstance(func, Type) if decorate: # Decoration is no-op assert profile(func) is func From 39451d98eda8f59d40c1da3a71d36acafa7ca345 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 1 Jun 2025 00:43:19 +0200 Subject: [PATCH 50/70] Fixed bad stub file --- line_profiler/profiler_mixin.pyi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/line_profiler/profiler_mixin.pyi b/line_profiler/profiler_mixin.pyi index 10c91f7e..3085a4a7 100644 --- a/line_profiler/profiler_mixin.pyi +++ b/line_profiler/profiler_mixin.pyi @@ -78,7 +78,7 @@ if TYPE_CHECKING: ... @__dict__.setter - def __dict__(self, dict: str | None) -> None: + def __dict__(self, dict: Dict[str, Any]) -> None: ... @property @@ -90,7 +90,7 @@ if TYPE_CHECKING: ... @__annotations__.setter - def __annotations__(self, annotations: str | None) -> None: + def __annotations__(self, annotations: Dict[str, Any]) -> None: ... @property From 9cd4ec9cceda3b344ba0f0f3b8b1521fdb380ab9 Mon Sep 17 00:00:00 2001 From: joncrall Date: Sun, 8 Jun 2025 16:27:41 -0400 Subject: [PATCH 51/70] Debugging --- line_profiler/_diagnostics.py | 30 ++++ line_profiler/_logger.py | 286 ++++++++++++++++++++++++++++++++++ 2 files changed, 316 insertions(+) create mode 100644 line_profiler/_diagnostics.py create mode 100644 line_profiler/_logger.py diff --git a/line_profiler/_diagnostics.py b/line_profiler/_diagnostics.py new file mode 100644 index 00000000..ad3830cb --- /dev/null +++ b/line_profiler/_diagnostics.py @@ -0,0 +1,30 @@ +""" +Global state initialized at import time. +Used for hidden arguments and developer features. +""" +from line_profiler import _logger +import os + + +def _boolean_environ(key): + """ + Args: + key (str) + + Returns: + bool + """ + value = os.environ.get(key, '').lower() + TRUTHY_ENVIRONS = {'true', 'on', 'yes', '1'} + return value in TRUTHY_ENVIRONS + + +DEBUG = _boolean_environ('LINE_PROFILER_DEBUG') +NO_EXEC = _boolean_environ('LINE_PROFILER_NO_EXEC') +KEEP_TEMPDIRS = _boolean_environ('LINE_PROFILER_KEEP_TEMPDIRS') + +# DEBUG_TEMPDIR = DEBUG or _boolean_environ('LINE_PROFILER_DEBUG_TEMPDIR') +# DEBUG_CORE = DEBUG or _boolean_environ('XDOCTEST_DEBUG_CORE') +# DEBUG_RUNNER = DEBUG or _boolean_environ('XDOCTEST_DEBUG_RUNNER') +# DEBUG_DOCTEST = DEBUG or _boolean_environ('XDOCTEST_DEBUG_DOCTEST') +log = _logger.Logger() diff --git a/line_profiler/_logger.py b/line_profiler/_logger.py new file mode 100644 index 00000000..1a203f63 --- /dev/null +++ b/line_profiler/_logger.py @@ -0,0 +1,286 @@ +""" +# Ported from kwutil +""" +# import os +import logging +from abc import ABC, abstractmethod +import sys +from logging import INFO, DEBUG, ERROR, WARNING, CRITICAL # NOQA + + +class _LogBackend(ABC): + """ + Abstract base class for our logger implementations. + """ + + def __init__(self, name): + self.name = name + + @abstractmethod + def debug(self, msg, *args, **kwargs): + pass + + @abstractmethod + def info(self, msg, *args, **kwargs): + pass + + @abstractmethod + def warning(self, msg, *args, **kwargs): + pass + + @abstractmethod + def error(self, msg, *args, **kwargs): + pass + + @abstractmethod + def critical(self, msg, *args, **kwargs): + pass + + +class _PrintLogBackend(_LogBackend): + """ + A simple print-based logger that falls back to print output if no logging configuration + is set up. + + Example: + >>> pl = _PrintLogBackend(name='print', level=INFO) + >>> pl.info('Hello %s', 'world') + Hello world + >>> pl.debug('Should not appear') + """ + + def __init__(self, name="", level=logging.INFO): + super().__init__(name) + self.level = level + + def isEnabledFor(self, level): + return level >= self.level + + def _log(self, level, msg, *args, **kwargs): + if self.isEnabledFor(level): + # Mimic logging formatting (ignoring extra kwargs for simplicity) + print(msg % args) + + def debug(self, msg, *args, **kwargs): + self._log(logging.DEBUG, msg, *args, **kwargs) + + def info(self, msg, *args, **kwargs): + self._log(logging.INFO, msg, *args, **kwargs) + + def warning(self, msg, *args, **kwargs): + self._log(logging.WARNING, msg, *args, **kwargs) + + def error(self, msg, *args, **kwargs): + self._log(logging.ERROR, msg, *args, **kwargs) + + def critical(self, msg, *args, **kwargs): + self._log(logging.CRITICAL, msg, *args, **kwargs) + + def configure(self, **kwargs): + """ + For compatability with _StdlibLogBackend, all arguments are ignored. + The print backend does not need to be configured. + """ + ... + + +class _StdlibLogBackend(_LogBackend): + """ + A wrapper for Python's standard logging.Logger. + + The constructor optionally adds a StreamHandler (to stdout) and/or a logging.FileHandler if file is specified. + + Example: + >>> import os + >>> import ubelt as ub + >>> import logging + >>> dpath = ub.Path.appdir('kwutil/test/logging').ensuredir() + >>> fpath = (dpath / 'test.log').delete() + >>> sl = _StdlibLogBackend('stdlib').configure( + >>> level=logging.INFO, + >>> stream={ + >>> 'format': '%(asctime)s : [stream] %(levelname)s : %(message)s', + >>> }, + >>> file={ + >>> 'path': fpath, + >>> 'format': '%(asctime)s : [file] %(levelname)s : %(message)s', + >>> } + >>> ) + >>> sl.info('Hello %s', 'world') + >>> # Check that the log file has been written to + >>> text = fpath.read_text() + >>> print(text) + >>> assert text.strip().endswith('Hello world') + """ + + def __init__(self, name): + super().__init__(name) + self.logger = logging.getLogger(name) + + def configure( + self, + level=None, + stream='auto', + file=None, + ): + """ + Configure the underlying stdlib logger. + + Parameters: + level: the logging level to set (e.g. logging.INFO) + stream: either a dict with configuration or a boolean/'auto' + - If dict, expected keys include 'format' + - If 'auto', the stream handler is enabled if no handlers are set + - If a boolean, True enables the stream handler. + file: either a dict with configuration or a path string. + - If dict, expected keys include 'path' and 'format' + - If a string, it is taken as the file path + + + Note: + For special attributes for the ``format`` argument of ``stream`` + and ``file`` see + https://docs.python.org/3/library/logging.html#logrecord-attributes + + Returns: + self (the configured _StdlibLogBackend instance) + """ + if level is not None: + self.logger.setLevel(level) + + # Default settings for file and stream handlers + fileinfo = { + 'path': None, + 'format': '%(asctime)s : [file] %(levelname)s : %(message)s' + } + streaminfo = { + '__enable__': None, # will be determined below + 'format': '%(levelname)s: %(message)s', + } + + # Update stream info if stream is a dict + if isinstance(stream, dict): + streaminfo.update(stream) + # If not specified otherwise, enable the stream handler. + if streaminfo.get('__enable__') is None: + streaminfo['__enable__'] = True + else: + # If stream is not a dict, treat it as a boolean or 'auto' + streaminfo['__enable__'] = stream + + # If stream is 'auto', enable stream only if no handlers are present. + if streaminfo['__enable__'] == 'auto': + streaminfo['__enable__'] = not bool(self.logger.handlers) + + # Update file info if file is a dict + if isinstance(file, dict): + fileinfo.update(file) + else: + fileinfo['path'] = file + + # Add a stream handler if enabled + if streaminfo['__enable__']: + streamformat = streaminfo.get('format') + sh = logging.StreamHandler(sys.stdout) + sh.setFormatter(logging.Formatter(streamformat)) + self.logger.addHandler(sh) + + # Add a file handler if a valid path is provided + path = fileinfo.get('path') + if path: + fileformat = fileinfo.get('format') + fh = logging.FileHandler(path) + fh.setFormatter(logging.Formatter(fileformat)) + self.logger.addHandler(fh) + return self + + # def _setup_handlers(self, stream, file): + # # Only add handlers if none exist, so as not to duplicate logs. + # if not self.logger.handlers: + + def debug(self, msg, *args, **kwargs): + kwargs['stacklevel'] = kwargs.get('stacklevel', 1) + 1 + self.logger.debug(msg, *args, **kwargs) + + def info(self, msg, *args, **kwargs): + kwargs['stacklevel'] = kwargs.get('stacklevel', 1) + 1 + self.logger.info(msg, *args, **kwargs) + + def warning(self, msg, *args, **kwargs): + kwargs['stacklevel'] = kwargs.get('stacklevel', 1) + 1 + self.logger.warning(msg, *args, **kwargs) + + def error(self, msg, *args, **kwargs): + kwargs['stacklevel'] = kwargs.get('stacklevel', 1) + 1 + self.logger.error(msg, *args, **kwargs) + + def critical(self, msg, *args, **kwargs): + kwargs['stacklevel'] = kwargs.get('stacklevel', 1) + 1 + self.logger.critical(msg, *args, **kwargs) + + +class Logger: + """ + The main Logger class that automatically selects the backend. + + If backend='auto' and a global logging configuration exists (i.e. logging.getLogger(name) has handlers), + it uses _StdlibLogBackend; otherwise, it falls back to _PrintLogBackend. + + Optional parameters: + - verbose: controls log level via an integer (0: CRITICAL, 1: INFO, 2: DEBUG, etc.) + - file: if provided, file logging is enabled (only used with _StdlibLogBackend) + - stream: if True, a logging.StreamHandler to stdout is added (only used with _StdlibLogBackend) + + Example: + >>> # With no global handlers, defaults to _PrintLogBackend + >>> logger = Logger('TestLogger', verbose=2, backend='auto') + >>> logger.info('Hello %s', 'world') + Hello world + >>> # Forcing use of _PrintLogBackend + >>> logger = Logger('TestLogger', verbose=2, backend='print') + >>> logger.debug('Debug %d', 123) + Debug 123 + >>> # Forcing use of Stdlib Logger + >>> logger = Logger('TestLogger', verbose=2, backend='stdlib') + >>> logger.debug('Debug %d', 123) + + Example: + >>> # Forcing use of Stdlib Logger + >>> logger = Logger('TestLogger', verbose=2, backend='stdlib').configure( + >>> stream={'format': '%(asctime)s : %(pathname)s:%(lineno)d %(funcName)s %(levelname)s : %(message)s'}) + >>> logger.debug('Debug %d', 123) + >>> logger.info('Hello %d', 123) + """ + def __init__(self, name="Logger", verbose=1, backend="auto", file=None, stream=True): + # Map verbose level to logging levels. If verbose > 1, show DEBUG, else INFO. + self.name = name + self.configure(verbose=verbose, backend=backend, file=file, stream=stream) + + def configure(self, backend='auto', verbose=1, file=None, stream=None): + name = self.name + kwargs = dict(file=file, stream=stream) + level = { + 0: logging.CRITICAL, + 1: logging.INFO, + 2: logging.DEBUG}.get(verbose, logging.DEBUG) + kwargs['level'] = level + if backend == "auto": + # Choose _StdlibLogBackend if a logger with handlers exists. + if logging.getLogger(name).handlers: + backend = 'stdlib' + else: + backend = 'print' + + if backend == "print": + backend_choice = _PrintLogBackend(name, level) + elif backend == "stdlib": + backend_choice = _StdlibLogBackend(name).configure(level, file=file, stream=stream) + else: + raise ValueError("Unsupported backend. Use 'auto', 'print', or 'stdlib'.") + self._backend = backend_choice + return self + + def __getattr__(self, attr): + # We should not need to modify stacklevel here as we are directly + # returning the backend function and not wrapping it. + return getattr(self._backend, attr) From 7b8b7efee0d9d4132e58bd6a5608fff7f391165e Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 9 Jun 2025 00:31:41 +0200 Subject: [PATCH 52/70] Using exit hooks for deleting tempfiles kernprof.py::_write_preimports() - Replaced `tempfile.TemporaryDirectory` context with explicit `try: ... except ...: ...` blocks so that: - If the temporary pre-import module is written but its execution failed, its deletion is deferred to an `atexit` hook so that the tempfile is preserved at traceback-formatting time - Otherwise, the tempfiles are scrubbed as soon as possible, e.g. when the pre-imports cannot be written or when they are successfully executed - Removed manual formatting of tracebacks tests/test_kernprof.py::test_kernprof_eager_preimport_bad_module() New test ensuring that the pre-import tempfile exists at traceback-formatting time --- kernprof.py | 57 ++++++++++++++++++++++++------------------ tests/test_kernprof.py | 38 ++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 25 deletions(-) diff --git a/kernprof.py b/kernprof.py index 76759634..db26bc99 100755 --- a/kernprof.py +++ b/kernprof.py @@ -128,6 +128,7 @@ def main(): To restore the old behavior, pass the :option:`!--no-preimports` flag. """ +import atexit import builtins import functools import os @@ -136,9 +137,9 @@ def main(): import asyncio # NOQA import concurrent.futures # NOQA import contextlib +import shutil import tempfile import time -import traceback import warnings from argparse import ArgumentError, ArgumentParser from datetime import datetime @@ -280,7 +281,6 @@ def _python_command(): """ Return a command that corresponds to :py:data:`sys.executable`. """ - import shutil for abbr in 'python', 'python3': if os.path.samefile(shutil.which(abbr), sys.executable): return abbr @@ -680,6 +680,11 @@ def _write_preimports(prof, options, exclude): """ Called by :py:func:`main()` to handle eager pre-imports; not to be invoked on its own. + + Notes + ----- + For the preservation of traceback messages, deletion of the + tempfiles created may be deferred to when the interpretor exits. """ from line_profiler.autoprofile.eager_preimports import ( is_dotted_path, propose_names, write_eager_import_module) @@ -694,14 +699,15 @@ def _write_preimports(prof, options, exclude): if is_dotted_path(target): modname = target else: - # Paths already normalized by `_normalize_profiling_targets()` + # Paths already normalized by + # `_normalize_profiling_targets()` if not os.path.exists(target): invalid_targets.append(target) continue if any(os.path.samefile(target, excluded) for excluded in exclude): # Ignore the script to be run in eager importing - # (`line_profiler.autoprofile.autoprofile.run()` will handle - # it) + # (`line_profiler.autoprofile.autoprofile.run()` will + # handle it) continue modname = modpath_to_modname(target, hide_init=False) if modname is None: # Not import-able @@ -723,20 +729,20 @@ def _write_preimports(prof, options, exclude): warnings.warn(msg) if not (filtered_targets or recurse_targets): return - # - We could've done everything in-memory with `io.StringIO` and - # `exec()`, but that results in indecipherable tracebacks should - # anything goes wrong; - # so we write to a tempfile and `execfile()` it - # - While this works theoretically for preserving traceback, the - # catch is that the tempfile will already have been deleted by the - # time the traceback is formatted; - # so we have to format the traceback and manually print the - # context before re-raising the error + # We could've done everything in-memory with `io.StringIO` and + # `exec()`, but that results in indecipherable tracebacks should + # anything goes wrong; + # so we write to a tempfile and `execfile()` it upgrade_profiler(prof) temp_mod_name = next( name for name in propose_names(['_kernprof_eager_preimports']) if name not in sys.modules) - with tempfile.TemporaryDirectory() as tmpdir: + # Don't use `tempfile.TemporaryDirectory` here, or the pre-import + # module will have been deleted by the time the traceback is + # formatted + tmpdir = tempfile.mkdtemp() + remove = functools.partial(shutil.rmtree, tmpdir, ignore_errors=True) + try: temp_mod_path = os.path.join(tmpdir, temp_mod_name + '.py') write_module = functools.partial( write_eager_import_module, filtered_targets, @@ -755,17 +761,18 @@ def _write_preimports(prof, options, exclude): else: with temp_file as fobj: write_module(stream=fobj) + except Exception: # Tempfile creation failed + remove() + raise + try: ns = {} # Use a fresh namespace - try: - execfile(temp_mod_path, ns, ns) - except Exception as e: - tb_lines = traceback.format_tb(e.__traceback__) - i_last_temp_frame = max( - i for i, line in enumerate(tb_lines) - if temp_mod_path in line) - print('\nContext:', ''.join(tb_lines[i_last_temp_frame:]), - end='', sep='\n', file=sys.stderr) - raise + execfile(temp_mod_path, ns, ns) + except BaseException: + # Defer deletion to after the traceback has been formatted + atexit.register(remove) + raise + else: # Delete the tempfiles ASAP + remove() def _main(options, module=False): diff --git a/tests/test_kernprof.py b/tests/test_kernprof.py index eed6f576..0aab2fa2 100644 --- a/tests/test_kernprof.py +++ b/tests/test_kernprof.py @@ -264,6 +264,44 @@ def main(): flags=re.MULTILINE)) == expect_match +def test_kernprof_eager_preimport_bad_module(): + """ + Test for the proper error messages when an error occurs in an + auto-generated pre-import module. + """ + bad_module = '''raise Exception('Boo')''' + with contextlib.ExitStack() as stack: + enter = stack.enter_context + tmpdir = enter(tempfile.TemporaryDirectory()) + temp_dpath = ub.Path(tmpdir) + (temp_dpath / 'my_bad_module.py').write_text(bad_module) + enter(ub.ChDir(tmpdir)) + python_path = os.environ.get('PYTHONPATH', '') + if python_path: + python_path = f'{python_path}:{tmpdir}' + else: + python_path = tmpdir + proc = ub.cmd(['kernprof', '-l', + # Add an eager pre-import target + '-pmy_bad_module', '-c', 'print(1)'], + env={**os.environ, 'PYTHONPATH': python_path}) + # Check that the traceback is preserved + print(proc.stdout) + print(proc.stderr, file=sys.stderr) + assert proc.returncode + assert 'import my_bad_module' in proc.stderr + assert bad_module in proc.stderr + # Check that the generated tempfiles are wiped + reverse_iter_lines = iter(reversed(proc.stderr.splitlines())) + next(line for line in reverse_iter_lines if 'import my_bad_module' in line) + tb_header = next(reverse_iter_lines).strip() + match = re.match('File ([\'"])(.+)\\1, line [0-9]+, in .*', tb_header) + assert match + tmp_mod = match.group(2) + assert not os.path.exists(tmp_mod) + assert not os.path.exists(os.path.dirname(tmp_mod)) + + class TestKernprof(unittest.TestCase): def test_enable_disable(self): From 08e2ab63b82850842f59283204e4e4f7e0c309f9 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 9 Jun 2025 02:12:30 +0200 Subject: [PATCH 53/70] More tempfile management kernprof.py main() - Moved tempfile managerment code here - Added note in docstring _touch_tempfile() New convenience function wrapping around `tempfile.mkstemp()` _write_tempfile() Updated function signature and implementation _write_preimports() Simplified implementation (because the cleanup code is moved outside to `main()` tests/test_kernprof.py test_kernprof_eager_preimport_bad_module() Updated docstring test_kernprof_bad_temp_script() New test analogous to the above checking that the temp script is available at traceback-formatting time --- kernprof.py | 128 ++++++++++++++++++++++++----------------- tests/test_kernprof.py | 39 ++++++++++++- 2 files changed, 112 insertions(+), 55 deletions(-) diff --git a/kernprof.py b/kernprof.py index db26bc99..4f2ac1c8 100755 --- a/kernprof.py +++ b/kernprof.py @@ -425,6 +425,11 @@ def pre_parse_single_arg_directive(args, flag, sep='--'): def main(args=None): """ Runs the command line interface + + Note: + To help with traceback formatting, the deletion of temporary + files created during execution may be deferred to when the + interpreter exits. """ def positive_float(value): val = float(value) @@ -638,13 +643,50 @@ def print_code_block_diagnostics( enter(contextlib.redirect_stdout(devnull)) if options.verbose < -2: # Suppress stderr enter(contextlib.redirect_stderr(devnull)) + # Instead of relying on `tempfile.TemporaryDirectory`, manually + # manage a tempdir to ensure that files exist at + # traceback-formatting time if needs be + options.tmpdir = tmpdir = tempfile.mkdtemp() + cleanup = functools.partial( + shutil.rmtree, tmpdir, ignore_errors=True, + ) if tempfile_source_and_content: - tmpdir = enter(tempfile.TemporaryDirectory()) - _write_tempfile(*tempfile_source_and_content, options, tmpdir) - _main(options, module) + try: + _write_tempfile(*tempfile_source_and_content, options) + except Exception: + # Tempfile creation failed, delete the tempdir ASAP + cleanup() + raise + try: + _main(options, module) + except BaseException: + # Defer deletion to after the traceback has been formatted + # if needs be + if os.listdir(tmpdir): + atexit.register(cleanup) + else: # Empty tempdir, just delete it + cleanup() + raise + else: # Execution succeeded, delete the tempdir ASAP + cleanup() + + +def _touch_tempfile(*args, **kwargs): + """ + Wrapper around :py:func:`tempfile.mkstemp()` which drops and closes + the integer handle (which we don't need and may cause issues on some + platforms). + """ + handle, path = tempfile.mkstemp(*args, **kwargs) + try: + os.close(handle) + except Exception: + os.remove(path) + raise + return path -def _write_tempfile(source, content, options, tmpdir): +def _write_tempfile(source, content, options): """ Called by :py:func:`main()` to handle :command:`kernprof -c` and :command:`kernprof -`; @@ -654,7 +696,7 @@ def _write_tempfile(source, content, options, tmpdir): file_prefix = f'kernprof-{source}' # Do what 3.14 does (#103998)... and also just to be user-friendly content = dedent(content) - fname = os.path.join(tmpdir, file_prefix + '.py') + fname = os.path.join(options.tmpdir, file_prefix + '.py') with open(fname, mode='w') as fobj: print(content, file=fobj) options.code_diagnostics(f'Wrote temporary script file to {fname!r}:', @@ -669,9 +711,9 @@ def _write_tempfile(source, content, options, tmpdir): # filename clash) if not options.outfile: extension = 'lprof' if options.line_by_line else 'prof' - _, options.outfile = tempfile.mkstemp(dir=os.curdir, - prefix=file_prefix + '-', - suffix='.' + extension) + options.outfile = _touch_tempfile(dir=os.curdir, + prefix=file_prefix + '-', + suffix='.' + extension) options.diagnostics( f'Using default output destination {options.outfile!r}') @@ -680,14 +722,9 @@ def _write_preimports(prof, options, exclude): """ Called by :py:func:`main()` to handle eager pre-imports; not to be invoked on its own. - - Notes - ----- - For the preservation of traceback messages, deletion of the - tempfiles created may be deferred to when the interpretor exits. """ from line_profiler.autoprofile.eager_preimports import ( - is_dotted_path, propose_names, write_eager_import_module) + is_dotted_path, write_eager_import_module) from line_profiler.autoprofile.util_static import modpath_to_modname from line_profiler.autoprofile.autoprofile import ( _extend_line_profiler_for_profiling_imports as upgrade_profiler) @@ -734,45 +771,30 @@ def _write_preimports(prof, options, exclude): # anything goes wrong; # so we write to a tempfile and `execfile()` it upgrade_profiler(prof) - temp_mod_name = next( - name for name in propose_names(['_kernprof_eager_preimports']) - if name not in sys.modules) - # Don't use `tempfile.TemporaryDirectory` here, or the pre-import - # module will have been deleted by the time the traceback is - # formatted - tmpdir = tempfile.mkdtemp() - remove = functools.partial(shutil.rmtree, tmpdir, ignore_errors=True) - try: - temp_mod_path = os.path.join(tmpdir, temp_mod_name + '.py') - write_module = functools.partial( - write_eager_import_module, filtered_targets, - recurse=recurse_targets) - temp_file = open(temp_mod_path, mode='w') - if options.verbose >= DIAGNOSITICS_VERBOSITY: - with StringIO() as sio: - write_module(stream=sio) - code = sio.getvalue() - with temp_file as fobj: - print(code, file=fobj) - options.code_diagnostics( - 'Wrote temporary module for pre-imports ' - f'to {temp_mod_path!r}:', - code) - else: - with temp_file as fobj: - write_module(stream=fobj) - except Exception: # Tempfile creation failed - remove() - raise - try: - ns = {} # Use a fresh namespace - execfile(temp_mod_path, ns, ns) - except BaseException: - # Defer deletion to after the traceback has been formatted - atexit.register(remove) - raise - else: # Delete the tempfiles ASAP - remove() + temp_mod_path = _touch_tempfile(dir=options.tmpdir, + prefix='kernprof-eager-preimports-', + suffix='.py') + write_module = functools.partial( + write_eager_import_module, filtered_targets, + recurse=recurse_targets) + temp_file = open(temp_mod_path, mode='w') + if options.verbose >= DIAGNOSITICS_VERBOSITY: + with StringIO() as sio: + write_module(stream=sio) + code = sio.getvalue() + with temp_file as fobj: + print(code, file=fobj) + options.code_diagnostics( + 'Wrote temporary module for pre-imports ' + f'to {temp_mod_path!r}:', + code) + else: + with temp_file as fobj: + write_module(stream=fobj) + ns = {} # Use a fresh namespace + execfile(temp_mod_path, ns, ns) + # Delete the tempfile ASAP if its execution succeeded + os.remove(temp_mod_path) def _main(options, module=False): diff --git a/tests/test_kernprof.py b/tests/test_kernprof.py index 0aab2fa2..ed3621de 100644 --- a/tests/test_kernprof.py +++ b/tests/test_kernprof.py @@ -2,6 +2,7 @@ import os import re import shlex +import subprocess import sys import tempfile import unittest @@ -266,8 +267,8 @@ def main(): def test_kernprof_eager_preimport_bad_module(): """ - Test for the proper error messages when an error occurs in an - auto-generated pre-import module. + Test for the preservation of the full traceback when an error occurs + in an auto-generated pre-import module. """ bad_module = '''raise Exception('Boo')''' with contextlib.ExitStack() as stack: @@ -302,6 +303,40 @@ def test_kernprof_eager_preimport_bad_module(): assert not os.path.exists(os.path.dirname(tmp_mod)) +@pytest.mark.parametrize('stdin', [True, False]) +def test_kernprof_bad_temp_script(stdin): + """ + Test for the preservation of the full traceback when an error occurs + in a temporary script supplied via `kernprof -c` or `kernprof -`. + """ + bad_script = '''1 / 0''' + with contextlib.ExitStack() as stack: + enter = stack.enter_context + enter(ub.ChDir(enter(tempfile.TemporaryDirectory()))) + if stdin: + proc = subprocess.run( + ['kernprof', '-'], + input=bad_script, capture_output=True, text=True) + else: + proc = subprocess.run(['kernprof', '-c', bad_script], + capture_output=True, text=True) + # Check that the traceback is preserved + print(proc.stdout) + print(proc.stderr, file=sys.stderr) + assert proc.returncode + assert '1 / 0' in proc.stderr + assert 'ZeroDivisionError' in proc.stderr + # Check that the generated tempfiles are wiped + reverse_iter_lines = iter(reversed(proc.stderr.splitlines())) + next(line for line in reverse_iter_lines if '1 / 0' in line) + tb_header = next(reverse_iter_lines).strip() + match = re.match('File ([\'"])(.+)\\1, line [0-9]+, in .*', tb_header) + assert match + tmp_script = match.group(2) + assert not os.path.exists(tmp_script) + assert not os.path.exists(os.path.dirname(tmp_script)) + + class TestKernprof(unittest.TestCase): def test_enable_disable(self): From abc681897d29b8402c2f669acac608f672696fea Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 9 Jun 2025 03:15:06 +0200 Subject: [PATCH 54/70] Don't write profiling data from tempfiles kernprof.py::_main() Instead of directly calling `prof.dump_stats()`, now filtering the stats so as not to include profiling data from tempfiles created in `kernprof.main()`; this removes the "Could not find file ..." error messages when the profile data is later read with `python -m line_profiler` tests/test_autoprofile.py::test_autoprofile_from_stdin() Updated to include check for the above --- kernprof.py | 27 ++++++++++++++++++++++++++- tests/test_autoprofile.py | 11 +++++++---- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/kernprof.py b/kernprof.py index 4f2ac1c8..b31d2b8e 100755 --- a/kernprof.py +++ b/kernprof.py @@ -828,6 +828,31 @@ def call_with_diagnostics(func, *args, **kwargs): options.code_diagnostics('Calling:', call) return func(*args, **kwargs) + def dump_filtered_stats(prof, filename): + import pickle + + tempfile_checks = {functools.partial(os.path.samefile, + os.path.join(dirname, fname)) + for dirname, _, fnames in os.walk(options.tmpdir) + for fname in fnames} + if not tempfile_checks: + return prof.dump_stats(filename) + # Filter the filenames to remove data from tempfiles, which will + # have been deleted by the time the results are viewed in a + # separate process + stats = prof.get_stats() + timings = stats.timings + for key in set(timings): + fname, *_ = key + try: + del_key = any(check(fname) for check in tempfile_checks) + except OSError: + del_key = True + if del_key: + del timings[key] + with open(filename, mode='wb') as fobj: + pickle.dump(stats, fobj, protocol=pickle.HIGHEST_PROTOCOL) + if not options.outfile: extension = 'lprof' if options.line_by_line else 'prof' options.outfile = f'{os.path.basename(options.script)}.{extension}' @@ -942,7 +967,7 @@ def call_with_diagnostics(func, *args, **kwargs): finally: if options.output_interval: rt.stop() - prof.dump_stats(options.outfile) + dump_filtered_stats(prof, options.outfile) options.message(f'Wrote profile results to {options.outfile!r}') if options.verbose > 0: if isinstance(prof, ContextualProfile): diff --git a/tests/test_autoprofile.py b/tests/test_autoprofile.py index 7f82f85b..040ec83d 100644 --- a/tests/test_autoprofile.py +++ b/tests/test_autoprofile.py @@ -586,14 +586,14 @@ def test_autoprofile_from_stdin( proc.check_returncode() outfile, = temp_dpath.glob(expected_outfile) + lp_cmd = [sys.executable, '-m', 'line_profiler', str(outfile)] + lp_proc = ub.cmd(lp_cmd) + lp_proc.check_returncode() if view: raw_output = proc.stdout else: - lp_cmd = [sys.executable, '-m', 'line_profiler', str(outfile)] - proc = ub.cmd(lp_cmd) - raw_output = proc.stdout + raw_output = lp_proc.stdout print(raw_output) - proc.check_returncode() assert ('Function: add_one' in raw_output) == prof_mod assert 'Function: add_two' not in raw_output @@ -601,6 +601,9 @@ def test_autoprofile_from_stdin( # If we're calling a separate process to view the results, the # script file will already have been deleted assert ('Function: main' in raw_output) == view + # Check that `main()` is scrubbed from the written file and doesn't + # result in spurious error messages + assert 'Could not find file' not in lp_proc.stdout @pytest.mark.parametrize( From 12ae3e2f00f6649ba6d4cebfb4e7acbdd9f10ef1 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 15 Jun 2025 03:32:43 +0200 Subject: [PATCH 55/70] Adaptation of TTsangSC#1 line_profiler/_diagnostics.py - Removed unused comments - Added default name (`'line_profiler'`) to `.log` line_profiler/_logger.py _LogBackend - Added class attribute `.backend` - Added abstract method `.configure()` _PrintLogBackend.configure() Now optionally taking `level` as the first argument like how `_StdlibLogBackend.configure()` does _StdlibLogBackend.configure() Now taking (and ignoring) arbitrary keyword arguments as specfied in `_LogBackend.configure()` Logger.configure() Updated and simplified implementation --- line_profiler/_diagnostics.py | 6 +---- line_profiler/_logger.py | 42 ++++++++++++++++++++++------------- 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/line_profiler/_diagnostics.py b/line_profiler/_diagnostics.py index ad3830cb..d368a41b 100644 --- a/line_profiler/_diagnostics.py +++ b/line_profiler/_diagnostics.py @@ -23,8 +23,4 @@ def _boolean_environ(key): NO_EXEC = _boolean_environ('LINE_PROFILER_NO_EXEC') KEEP_TEMPDIRS = _boolean_environ('LINE_PROFILER_KEEP_TEMPDIRS') -# DEBUG_TEMPDIR = DEBUG or _boolean_environ('LINE_PROFILER_DEBUG_TEMPDIR') -# DEBUG_CORE = DEBUG or _boolean_environ('XDOCTEST_DEBUG_CORE') -# DEBUG_RUNNER = DEBUG or _boolean_environ('XDOCTEST_DEBUG_RUNNER') -# DEBUG_DOCTEST = DEBUG or _boolean_environ('XDOCTEST_DEBUG_DOCTEST') -log = _logger.Logger() +log = _logger.Logger('line_profiler') diff --git a/line_profiler/_logger.py b/line_profiler/_logger.py index 1a203f63..83fac0fb 100644 --- a/line_profiler/_logger.py +++ b/line_profiler/_logger.py @@ -5,6 +5,7 @@ import logging from abc import ABC, abstractmethod import sys +from typing import ClassVar from logging import INFO, DEBUG, ERROR, WARNING, CRITICAL # NOQA @@ -12,10 +13,19 @@ class _LogBackend(ABC): """ Abstract base class for our logger implementations. """ + backend: ClassVar[str] def __init__(self, name): self.name = name + @abstractmethod + def configure(self, *args, **kwarg): + """ + Note: + Implementations should take the arguments it needs and + return the instance. + """ + @abstractmethod def debug(self, msg, *args, **kwargs): pass @@ -48,6 +58,7 @@ class _PrintLogBackend(_LogBackend): Hello world >>> pl.debug('Should not appear') """ + backend = 'print' def __init__(self, name="", level=logging.INFO): super().__init__(name) @@ -76,12 +87,10 @@ def error(self, msg, *args, **kwargs): def critical(self, msg, *args, **kwargs): self._log(logging.CRITICAL, msg, *args, **kwargs) - def configure(self, **kwargs): - """ - For compatability with _StdlibLogBackend, all arguments are ignored. - The print backend does not need to be configured. - """ - ... + def configure(self, level=None, **_): + if level is not None: + self.level = level + return self class _StdlibLogBackend(_LogBackend): @@ -112,6 +121,7 @@ class _StdlibLogBackend(_LogBackend): >>> print(text) >>> assert text.strip().endswith('Hello world') """ + backend = 'stdlib' def __init__(self, name): super().__init__(name) @@ -122,6 +132,7 @@ def configure( level=None, stream='auto', file=None, + **_, ): """ Configure the underlying stdlib logger. @@ -259,25 +270,24 @@ def __init__(self, name="Logger", verbose=1, backend="auto", file=None, stream=T def configure(self, backend='auto', verbose=1, file=None, stream=None): name = self.name kwargs = dict(file=file, stream=stream) - level = { + kwargs['level'] = { 0: logging.CRITICAL, 1: logging.INFO, 2: logging.DEBUG}.get(verbose, logging.DEBUG) - kwargs['level'] = level if backend == "auto": # Choose _StdlibLogBackend if a logger with handlers exists. if logging.getLogger(name).handlers: backend = 'stdlib' else: backend = 'print' - - if backend == "print": - backend_choice = _PrintLogBackend(name, level) - elif backend == "stdlib": - backend_choice = _StdlibLogBackend(name).configure(level, file=file, stream=stream) - else: - raise ValueError("Unsupported backend. Use 'auto', 'print', or 'stdlib'.") - self._backend = backend_choice + try: + Backend = {'print': _PrintLogBackend, + 'stdlib': _StdlibLogBackend}[backend] + except KeyError: + raise ValueError( + "Unsupported backend. " + "Use 'auto', 'print', or 'stdlib'.") from None + self._backend = Backend(name).configure(**kwargs) return self def __getattr__(self, attr): From 6488a9a3eeb96d514f9eac44883842872344a847 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 15 Jun 2025 04:27:07 +0200 Subject: [PATCH 56/70] Integration between TTsangSC#1 and #2 kernprof.py _restore - Refactored from `_restore_list` - New class methods `.sequence()`, `.mapping()`, and `.instance_dict()` for restoring various objects _remove() New utility function for deleting files and directories main() "Developer mode" options (note: all of the below should be considered implementational details) - Refactored internal methods for printing messages and diagnostics to use `line_profiler._diagnostics.log` - Now outputting diagnostics regardless of verbosity level if `line_profiler._diagnostics.DEBUG` - Now keeping the temporary files if `line_profiler._diagnostics.KEEP_TEMPDIRS` - Now not executing any source file (setup, pre-imports, profiled code) nor writing profile output if `line_profiler._diagnostics.NO_EXEC` --- kernprof.py | 291 +++++++++++++++++++++++++++++++++++----------------- 1 file changed, 198 insertions(+), 93 deletions(-) diff --git a/kernprof.py b/kernprof.py index b31d2b8e..777949be 100755 --- a/kernprof.py +++ b/kernprof.py @@ -144,7 +144,9 @@ def main(): from argparse import ArgumentError, ArgumentParser from datetime import datetime from io import StringIO +from operator import methodcaller from runpy import run_module +from pathlib import Path from pprint import pformat from shlex import quote from textwrap import indent, dedent @@ -162,6 +164,8 @@ def main(): from profile import Profile # type: ignore[assignment,no-redef] from line_profiler.profiler_mixin import ByCountProfilerMixin +from line_profiler._logger import Logger +from line_profiler import _diagnostics as diagnostics DIAGNOSITICS_VERBOSITY = 2 @@ -316,39 +320,24 @@ def find(path): return list(results) -class _restore_list: +class _restore: """ - Restore a list like :py:data:`sys.path` after running code which - potentially modifies it. - - Example - ------- - >>> l = [1, 2, 3] - >>> - >>> - >>> with _restore_list(l): - ... print(l) - ... l.append(4) - ... print(l) - ... l[:] = 5, 6 - ... print(l) - ... - [1, 2, 3] - [1, 2, 3, 4] - [5, 6] - >>> l - [1, 2, 3] + Restore a collection like :py:data:`sys.path` after running code + which potentially modifies it. """ - def __init__(self, lst): - self.lst = lst + def __init__(self, obj, getter, setter): + self.obj = obj + self.setter = setter + self.getter = getter self.old = None def __enter__(self): assert self.old is None - self.old = self.lst.copy() + self.old = self.getter(self.obj) def __exit__(self, *_, **__): - self.old, self.lst[:] = None, self.old + self.setter(self.obj, self.old) + self.old = None def __call__(self, func): @functools.wraps(func) @@ -358,6 +347,87 @@ def wrapper(*args, **kwargs): return wrapper + @classmethod + def sequence(cls, seq): + """ + Example + ------- + >>> l = [1, 2, 3] + >>> + >>> with _restore.sequence(l): + ... print(l) + ... l.append(4) + ... print(l) + ... l[:] = 5, 6 + ... print(l) + ... + [1, 2, 3] + [1, 2, 3, 4] + [5, 6] + >>> l + [1, 2, 3] + """ + def set_list(orig, copy): + orig[:] = copy + + return cls(seq, methodcaller('copy'), set_list) + + @classmethod + def mapping(cls, mpg): + """ + Example + ------- + >>> d = {1: 2} + >>> + >>> with _restore.mapping(d): + ... print(d) + ... d[2] = 3 + ... print(d) + ... d.clear() + ... d.update({1: 4, 3: 5}) + ... print(d) + ... + {1: 2} + {1: 2, 2: 3} + {1: 4, 3: 5} + >>> d + {1: 2} + """ + def set_mapping(orig, copy): + orig.clear() + orig.update(copy) + + return cls(mpg, methodcaller('copy'), set_mapping) + + @classmethod + def instance_dict(cls, obj): + """ + Example + ------- + >>> class Obj: + ... def __init__(self, x, y): + ... self.x, self.y = x, y + ... + ... def __repr__(self): + ... return 'Obj({0.x!r}, {0.y!r})'.format(self) + ... + >>> + >>> obj = Obj(1, 2) + >>> + >>> with _restore.instance_dict(obj): + ... print(obj) + ... obj.x, obj.y, obj.z = 4, 5, 6 + ... print(obj, obj.z) + ... + Obj(1, 2) + Obj(4, 5) 6 + >>> obj + Obj(1, 2) + >>> hasattr(obj, 'z') + False + """ + return cls.mapping(vars(obj)) + def pre_parse_single_arg_directive(args, flag, sep='--'): """ @@ -420,8 +490,9 @@ def pre_parse_single_arg_directive(args, flag, sep='--'): return args[:i_flag], args[i_flag + 1], args[i_flag + 2:] -@_restore_list(sys.argv) -@_restore_list(sys.path) +@_restore.sequence(sys.argv) +@_restore.sequence(sys.path) +@_restore.instance_dict(diagnostics) def main(args=None): """ Runs the command line interface @@ -437,34 +508,27 @@ def positive_float(value): raise ArgumentError return val - def print_message(threshold=0, /, *args, print=print, **kwargs): - if isinstance(threshold, str): - args = [threshold, *args] - threshold = 0 - if options.verbose < threshold: - return - print(*args, **kwargs) - def print_diagnostics(*args, **kwargs): - if options.rich: - from rich.console import Console - from rich.markup import escape + with StringIO() as sio: + if options.rich and simple_logging: + from rich.console import Console + from rich.markup import escape - printer = Console().print - else: - escape = str - printer = print + printer = Console(file=sio, force_terminal=True).print + else: + escape = str + printer = functools.partial(print, file=sio) - if args: - now = datetime.now().isoformat(sep=' ', timespec='seconds') - args = ['{} {}'.format(escape(f'[kernprof {now}]'), args[0]), - *args[1:]] - kwargs['print'] = printer - print_message(DIAGNOSITICS_VERBOSITY, *args, **kwargs) + if args and simple_logging: + now = datetime.now().isoformat(sep=' ', timespec='seconds') + args = ['{} {}'.format(escape(f'[kernprof {now}]'), args[0]), + *args[1:]] + printer(*args, **kwargs) + logger.debug(sio.getvalue()) def print_code_block_diagnostics( header, code, *, line_numbers=True, **kwargs): - if options.rich: + if options.rich and simple_logging: from rich.syntax import Syntax code_repr = Syntax(code, 'python', line_numbers=line_numbers) @@ -487,6 +551,9 @@ def print_code_block_diagnostics( args.append('') print_diagnostics(*args, **kwargs) + def no_op(*_, **__) -> None: + pass + create_parser = functools.partial( ArgumentParser, description='Run and profile a python script.') @@ -594,6 +661,9 @@ def print_code_block_diagnostics( # Hand off to the dummy parser if necessary to generate the help # text options = SimpleNamespace(**vars(real_parser.parse_args(args))) + # TODO: make a flag later where appropriate + options.dryrun = diagnostics.NO_EXEC + options.rm = no_op if diagnostics.KEEP_TEMPDIRS else _remove if help_parser and getattr(options, 'help', False): help_parser.print_help() exit() @@ -615,9 +685,23 @@ def print_code_block_diagnostics( # Handle output options.verbose -= options.quiet - options.message = (print_message - if options.verbose < DIAGNOSITICS_VERBOSITY else - print_diagnostics) + options.debug = (diagnostics.DEBUG + or options.verbose >= DIAGNOSITICS_VERBOSITY) + logger_kwargs = {'name': 'kernprof'} + if options.debug: + logger_kwargs['verbose'] = 2 + elif options.verbose > -1: + logger_kwargs['verbose'] = 1 + else: + logger_kwargs['verbose'] = 0 + # Also consume log items written from other `line_profiler` + # components + logger = diagnostics.log = Logger(**logger_kwargs) + simple_logging = logger.backend == 'print' + if options.debug: + options.message = print_diagnostics + else: + options.message = logger.info options.diagnostics = print_diagnostics options.code_diagnostics = print_code_block_diagnostics if options.rich: @@ -648,7 +732,7 @@ def print_code_block_diagnostics( # traceback-formatting time if needs be options.tmpdir = tmpdir = tempfile.mkdtemp() cleanup = functools.partial( - shutil.rmtree, tmpdir, ignore_errors=True, + options.rm, tmpdir, recursive=True, missing_ok=True, ) if tempfile_source_and_content: try: @@ -778,7 +862,7 @@ def _write_preimports(prof, options, exclude): write_eager_import_module, filtered_targets, recurse=recurse_targets) temp_file = open(temp_mod_path, mode='w') - if options.verbose >= DIAGNOSITICS_VERBOSITY: + if options.debug: with StringIO() as sio: write_module(stream=sio) code = sio.getvalue() @@ -791,10 +875,22 @@ def _write_preimports(prof, options, exclude): else: with temp_file as fobj: write_module(stream=fobj) - ns = {} # Use a fresh namespace - execfile(temp_mod_path, ns, ns) + if not options.dryrun: + ns = {} # Use a fresh namespace + execfile(temp_mod_path, ns, ns) # Delete the tempfile ASAP if its execution succeeded - os.remove(temp_mod_path) + options.rm(temp_mod_path) + + +def _remove(path, *, recursive=False, missing_ok=False): + path = Path(path) + if path.is_dir(): + if recursive: + shutil.rmtree(path, ignore_errors=missing_ok) + else: + path.rmdir() + else: + path.unlink(missing_ok=missing_ok) def _main(options, module=False): @@ -804,28 +900,31 @@ def _main(options, module=False): not to be invoked on its own. """ def call_with_diagnostics(func, *args, **kwargs): - if options.verbose < DIAGNOSITICS_VERBOSITY: - return func(*args, **kwargs) - if isinstance(func, MethodType): - obj = func.__self__ - func_repr = ('{0.__module__}.{0.__qualname__}(...).{1.__name__}' - .format(type(obj), func.__func__)) - else: - func_repr = '{0.__module__}.{0.__qualname__}'.format(func) - args_repr = dedent(' ' + pformat(args)[len('['):-len(']')]) - kwargs_repr = dedent( - ' ' * len('namespace(') - + pformat(SimpleNamespace(**kwargs))[len('namespace('):-len(')')]) - if args_repr and kwargs_repr: - all_args_repr = f'{args_repr},\n{kwargs_repr}' - else: - all_args_repr = args_repr or kwargs_repr - if all_args_repr: - call = '{}(\n{})'.format( - func_repr, indent(all_args_repr, ' ')) - else: - call = func_repr + '()' - options.code_diagnostics('Calling:', call) + if options.debug: + if isinstance(func, MethodType): + obj = func.__self__ + func_repr = ( + '{0.__module__}.{0.__qualname__}(...).{1.__name__}' + .format(type(obj), func.__func__)) + else: + func_repr = '{0.__module__}.{0.__qualname__}'.format(func) + args_repr = dedent(' ' + pformat(args)[len('['):-len(']')]) + lprefix = len('namespace(') + kwargs_repr = dedent( + ' ' * lprefix + + pformat(SimpleNamespace(**kwargs))[lprefix:-len(')')]) + if args_repr and kwargs_repr: + all_args_repr = f'{args_repr},\n{kwargs_repr}' + else: + all_args_repr = args_repr or kwargs_repr + if all_args_repr: + call = '{}(\n{})'.format( + func_repr, indent(all_args_repr, ' ')) + else: + call = func_repr + '()' + options.code_diagnostics('Calling:', call) + if options.dryrun: + return return func(*args, **kwargs) def dump_filtered_stats(prof, filename): @@ -876,7 +975,8 @@ def dump_filtered_stats(prof, filename): ns = {'__file__': setup_file, '__name__': '__main__'} options.diagnostics( f'Executing file {setup_file!r} as pre-profiling setup') - execfile(setup_file, ns, ns) + if not options.dryrun: + execfile(setup_file, ns, ns) if options.line_by_line: import line_profiler @@ -929,18 +1029,18 @@ def dump_filtered_stats(prof, filename): exclude = set() if module else {script_file} _write_preimports(prof, options, exclude) - if options.output_interval: + use_timer = options.output_interval and not options.dryrun + if use_timer: rt = RepeatedTimer(max(options.output_interval, 1), prof.dump_stats, options.outfile) original_stdout = sys.stdout - if options.output_interval: + if use_timer: rt = RepeatedTimer(max(options.output_interval, 1), prof.dump_stats, options.outfile) try: try: - execfile_ = execfile - rmod_ = functools.partial(run_module, - run_name='__main__', alter_sys=True) + rmod = functools.partial(run_module, + run_name='__main__', alter_sys=True) ns = {'__file__': script_file, '__name__': '__main__', - 'execfile_': execfile_, 'rmod_': rmod_, + 'execfile': execfile, 'rmod': rmod, 'prof': prof} if options.prof_mod and options.line_by_line: from line_profiler.autoprofile import autoprofile @@ -951,25 +1051,30 @@ def dump_filtered_stats(prof, filename): profile_imports=options.prof_imports, as_module=module is not None) elif module and options.builtin: - call_with_diagnostics(rmod_, options.script, ns) + call_with_diagnostics(rmod, options.script, ns) elif options.builtin: call_with_diagnostics(execfile, script_file, ns, ns) elif module: call_with_diagnostics( - prof.runctx, f'rmod_({options.script!r}, globals())', + prof.runctx, f'rmod({options.script!r}, globals())', ns, ns) else: call_with_diagnostics( - prof.runctx, f'execfile_({script_file!r}, globals())', + prof.runctx, f'execfile({script_file!r}, globals())', ns, ns) except (KeyboardInterrupt, SystemExit): pass finally: - if options.output_interval: + if use_timer: rt.stop() - dump_filtered_stats(prof, options.outfile) - options.message(f'Wrote profile results to {options.outfile!r}') - if options.verbose > 0: + if not options.dryrun: + dump_filtered_stats(prof, options.outfile) + options.message( + ('Profile results would have been written to ' + if options.dryrun else + 'Wrote profile results ') + + f'to {options.outfile!r}') + if options.verbose > 0 and not options.dryrun: if isinstance(prof, ContextualProfile): prof.print_stats() else: From 4f81a8c54815f332aadb145f6172eb9c37cb5743 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 15 Jun 2025 08:26:25 +0200 Subject: [PATCH 57/70] Separate static and dynamic pre-import code paths line_profiler/autoprofile/eager_preimports.py[i] __all__ New module attribute split_dotted_path() Added new argument `static` for choosing whether to use static analysis (`line_profiler.autoprofile.util_static.modname_to_modpath()`) or the import system (`importlib.util.find_spec()`) to resolve dotted paths resolve_profiling_targets() Split from `write_eager_import_module()` for easier testing write_eager_import_module() - Fixed malformed RST in docstring - Added new argument `static` for choosing whether to use static analysis or the import system to: - Resolve dotted paths (see `split_dotted_path()`), and - Find subpackages and -modules (`~.util_static.package_modpaths()` vs `pkgutil.walk_packages()`) --- line_profiler/autoprofile/eager_preimports.py | 189 +++++++++++++----- .../autoprofile/eager_preimports.pyi | 21 +- 2 files changed, 158 insertions(+), 52 deletions(-) diff --git a/line_profiler/autoprofile/eager_preimports.py b/line_profiler/autoprofile/eager_preimports.py index 333ba9fb..2e184046 100644 --- a/line_profiler/autoprofile/eager_preimports.py +++ b/line_profiler/autoprofile/eager_preimports.py @@ -5,14 +5,19 @@ import ast import functools import itertools +from collections import namedtuple from collections.abc import Collection from keyword import iskeyword from importlib.util import find_spec -from os.path import isdir from pkgutil import walk_packages from textwrap import dedent, indent as indent_ from warnings import warn -from .util_static import modname_to_modpath +from .util_static import ( + modname_to_modpath, modpath_to_modname, package_modpaths) + + +__all__ = ('is_dotted_path', 'split_dotted_path', + 'resolve_profiling_targets', 'write_eager_import_module') def is_dotted_path(obj): @@ -50,13 +55,16 @@ def get_expression(obj): return None -def split_dotted_path(dotted_path): +def split_dotted_path(dotted_path, static=True): """ Arguments: dotted_path (str): Dotted path indicating an import target (module, package, or a ``from ... import ...``-able name under that), or an object accessible via (chained) attribute access thereon + static (bool): + Whether to use static analysis (true) or the import system + (false) to resolve targets Returns: module, target (tuple[str, Union[str, None]]): @@ -76,6 +84,11 @@ def split_dotted_path(dotted_path): ModuleNotFoundError If a matching module cannot be found + Note: + ``static=False`` can cause the ancestor module objects of + ``dotted_path`` to be imported (and hence alter the state of + :py:data:`sys.modules`. + Example: >>> split_dotted_path('importlib.util.find_spec') ('importlib.util', 'find_spec') @@ -103,14 +116,15 @@ def split_dotted_path(dotted_path): '(string of period-joined identifiers)') chunks = dotted_path.split('.') checked_locs = [] + check = modname_to_modpath if static else find_spec for slicing_point in range(len(chunks), 0, -1): module = '.'.join(chunks[:slicing_point]) target = '.'.join(chunks[slicing_point:]) or None try: - spec = find_spec(module) + result = check(module) except ImportError: - spec = None - if spec is None: + result = None + if result is None: checked_locs.append(module) continue return module, target @@ -125,7 +139,7 @@ def strip(s): class LoadedNameFinder(ast.NodeVisitor): """ - Find the names loaded in an AST. A name is considered to be loaded + Find the names loaded in an AST. A name is considered to be loaded if it appears with the context :py:class:`ast.Load()` and is not an argument of any surrounding function-definition contexts (``def func(...): ...``, ``async def func(...): ...``, or @@ -221,7 +235,99 @@ def propose_names(prefixes): yield pattern(prefix, i) +def resolve_profiling_targets(dotted_paths, static=True, recurse=False): + """ + Arguments: + dotted_paths (Collection[str]): + Dotted paths (strings of period-joined identifiers) + indicating what should be profiled + static (bool): + Whether to use static analysis (true) or the import system + (false) to resolve targets + recurse (Union[Collection[str], bool]): + Dotted paths (strings of period-joined identifiers) + indicating the profiling targets that should be recursed + into if they are packages; + can also be a boolean value, indicating: + + :py:const:`True` + Recurse into any entry in ``dotted_paths`` that is a + package + :py:const:`False` + Don't recurse into any entry + + Returns: + result (ResolvedResult): + 3-named-tuple with the following fields/items: + + ``.targets`` (dict[str, set[str | None]]): + Mapping from module names to the names of the attributes + therein that should be passed to the profiler; + if the attribute name is :py:const:`None`, the whole + module is to be passed to the profiler + ``.indirect`` (set[str]): + Set of subpackage/-module names included only via + ``recurse``-ing into packages (i.e. not directly in + either ``dotted_paths`` or ``recurse``) + ``.unresolved`` (list[str]): + List of unresolved profiling targets, i.e. those which + cannot be resolved into a module part and an attribute + part by :py:func:`~.split_dotted_path` + + Note: + ``static=False`` can cause the ancestor module objects of + ``dotted_paths`` and ``recurse`` to be imported (and hence alter + the state of :py:data:`sys.modules`. + """ + def walk_packages_static(pkg): + # Note: this probably can't handle namespace packages + path = modname_to_modpath(pkg) + if not path: + return + for subpath in package_modpaths(path, with_pkg=True): + submod = modpath_to_modname(subpath) + if submod: + yield submod + + def walk_packages_import_sys(pkg): + spec = find_spec(pkg) + if not spec: + return + paths = spec.submodule_search_locations or [] + if not paths: + return + for info in walk_packages(paths, prefix=pkg + '.'): + yield info.name + + dotted_paths = set(dotted_paths) + if isinstance(recurse, Collection): + recurse = set(recurse) + else: + recurse = dotted_paths if recurse else set() + dotted_paths |= recurse + indirect_submods = set() + + all_targets = {} + unknown_locs = [] + split_path = functools.partial(split_dotted_path, static=static) + walk = walk_packages_static if static else walk_packages_import_sys + for path in sorted(set(dotted_paths)): + try: + module, target = split_path(path) + except ModuleNotFoundError: + unknown_locs.append(path) + continue + all_targets.setdefault(module, set()).add(target) + if path in recurse and target is None: + for submod in walk(path): + all_targets.setdefault(submod, set()).add(None) + indirect_submods.add(submod) + indirect_submods -= dotted_paths + return ResolvedResult(all_targets, indirect_submods, unknown_locs) + + def write_eager_import_module(dotted_paths, stream=None, *, + static=True, recurse=False, adder='profile.add_imported_function_or_module', indent=' '): @@ -235,6 +341,9 @@ def write_eager_import_module(dotted_paths, stream=None, *, stream (Union[TextIO, None]): Optional text-mode writable file object to which to write the module + static (bool): + Whether to use static analysis (true) or the import system + (false) to resolve targets recurse (Union[Collection[str], bool]): Dotted paths (strings of period-joined identifiers) indicating the profiling targets that should be recursed @@ -263,13 +372,12 @@ def write_eager_import_module(dotted_paths, stream=None, *, Raises: TypeError - * If ``adder`` and ``indent`` are not strings - * If ``dotted_paths`` is not a collection of dotted paths + If ``adder`` and ``indent`` are not strings, **OR** + if ``dotted_paths`` is not a collection of dotted paths ValueError - * If ``adder`` is a non-single-line string or is not - parsable to a single expression - * If ``indent`` isn't single-line, non-empty, and - whitespace + If ``adder`` is a non-single-line string or is not parsable + to a single expression, **OR** + if ``indent`` isn't single-line, non-empty, and whitespace Example: >>> import io @@ -331,6 +439,11 @@ def write_eager_import_module(dotted_paths, stream=None, *, >>> assert (record[0].message.args[0] ... == ("1 import target cannot be resolved: " ... "['foo.bar']")) + + Note: + ``static=False`` can cause the ancestor module objects of + ``dotted_paths`` and ``recurse`` to be imported (and hence alter + the state of :py:data:`sys.modules`. """ if not isinstance(adder, str): AdderError = TypeError @@ -377,49 +490,21 @@ def write_eager_import_module(dotted_paths, stream=None, *, if name not in forbidden_names) # Figure out the import targets to profile - dotted_paths = set(dotted_paths) - if isinstance(recurse, Collection): - recurse = set(recurse) - else: - recurse = dotted_paths if recurse else set() - dotted_paths |= recurse - paths_added_by_recursion = set() - - imports = {} - unknown_locs = [] - for path in sorted(set(dotted_paths)): - try: - module, target = split_dotted_path(path) - except ModuleNotFoundError: - unknown_locs.append(path) - continue - if path in recurse and target is None: - recurse_root = modname_to_modpath(path, hide_init=True) - if recurse_root and not isdir(recurse_root): - recurse_root = None - else: # Not a recurse target nor a module - recurse_root = None - imports.setdefault(module, set()).add(target) - # FIXME: how do we handle namespace packages? - if recurse_root is None: - continue - for info in walk_packages([recurse_root], prefix=module + '.'): - imports.setdefault(info.name, set()).add(None) - paths_added_by_recursion.add(info.name) - paths_added_by_recursion -= dotted_paths + resolved = resolve_profiling_targets( + dotted_paths, static=static, recurse=recurse) # Warn against failed imports - if unknown_locs: + if resolved.unresolved: msg = '{} import target{} cannot be resolved: {!r}'.format( - len(unknown_locs), - '' if len(unknown_locs) == 1 else 's', - unknown_locs) + len(resolved.unresolved), + '' if len(resolved.unresolved) == 1 else 's', + resolved.unresolved) warn(msg, stacklevel=2) # Do the imports and add them with `adder` write = functools.partial(print, file=stream) write(f'{adder_name} = {adder} # noqa: F821\n{failures_name} = []') - for i, (module, targets) in enumerate(imports.items()): + for i, (module, targets) in enumerate(resolved.targets.items()): assert targets # Write one more empty line so that the imports are separated # from the preambles by 2 lines @@ -427,7 +512,7 @@ def write_eager_import_module(dotted_paths, stream=None, *, write() # Allow arbitrary errors for modules that are only added # indirectly (via descent/recursion) - if module in paths_added_by_recursion: + if module in resolved.indirect: allowed_error = 'Exception' else: allowed_error = 'ImportError' @@ -464,7 +549,7 @@ def write_eager_import_module(dotted_paths, stream=None, *, for chunk in chunks: write(indent_(chunk, indent)) # Issue a warning if any of the targets doesn't exist - if imports: + if resolved.targets: write('\n') write(strip(f""" if {failures_name}: @@ -476,3 +561,7 @@ def write_eager_import_module(dotted_paths, stream=None, *, {indent * 2}{failures_name}) {indent}warnings.warn(msg, stacklevel=2) """)) + + +ResolvedResult = namedtuple('ResolvedResult', + ('targets', 'indirect', 'unresolved')) diff --git a/line_profiler/autoprofile/eager_preimports.pyi b/line_profiler/autoprofile/eager_preimports.pyi index c8b7b29a..756a6b7b 100644 --- a/line_profiler/autoprofile/eager_preimports.pyi +++ b/line_profiler/autoprofile/eager_preimports.pyi @@ -1,6 +1,8 @@ import ast from typing import ( - Any, Collection, Generator, List, Set, TextIO, Tuple, Union) + Any, Union, + Collection, Dict, Generator, List, NamedTuple, Set, Tuple, + TextIO) def is_dotted_path(obj: Any) -> bool: @@ -11,7 +13,8 @@ def get_expression(obj: Any) -> Union[ast.Expression, None]: ... -def split_dotted_path(dotted_path: str) -> Tuple[str, Union[str, None]]: +def split_dotted_path( + dotted_path: str, static: bool = True) -> Tuple[str, Union[str, None]]: ... @@ -42,9 +45,23 @@ def propose_names(prefixes: Collection[str]) -> Generator[str, None, None]: ... +def resolve_profiling_targets( + dotted_paths: Collection[str], + static: bool = True, + recurse: Union[Collection[str], bool] = False) -> 'ResolvedResult': + ... + + def write_eager_import_module( dotted_paths: Collection[str], stream: Union[TextIO, None] = None, *, + static: bool = True, recurse: Union[Collection[str], bool] = False, adder: str = 'profile.add_imported_function_or_module', indent: str = ' ') -> None: ... + + +class ResolvedResult(NamedTuple): + targets: Dict[str, Set[Union[str, None]]] + indirect: Set[str] + unresolved: List[str] From caa3b201da1d93168dcbd8af0da8b990438b5d4b Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 15 Jun 2025 10:07:30 +0200 Subject: [PATCH 58/70] Updated tests for eager pre-imports tests/test_eager_preimports.py sample_package() - Renamed from `sample_module()` - Simplified implementation preserve_sys_state(), sample_namespace_package() New fixtures gen_names() New utility function test_{split_dotted_path,resolve_profiling_targets}_staticity() New tests for how the `static` parameter influences the behavior of `line_profiler.autoprofile.eager_preimports.split_dotted_path()` and `.resolve_profiling_targets()` --- tests/test_eager_preimports.py | 113 +++++++++++++++++++++++++-------- 1 file changed, 86 insertions(+), 27 deletions(-) diff --git a/tests/test_eager_preimports.py b/tests/test_eager_preimports.py index 681745af..ba5d344d 100644 --- a/tests/test_eager_preimports.py +++ b/tests/test_eager_preimports.py @@ -8,6 +8,7 @@ import subprocess import sys from contextlib import ExitStack +from functools import partial from pathlib import Path from operator import methodcaller from runpy import run_path @@ -27,7 +28,7 @@ HAS_FLAKE8 = True from line_profiler.autoprofile.eager_preimports import ( - write_eager_import_module as write_module) + split_dotted_path, resolve_profiling_targets, write_eager_import_module) def write(path: Path, content: Optional[str] = None) -> None: @@ -38,19 +39,32 @@ def write(path: Path, content: Optional[str] = None) -> None: path.write_text(dedent(content).strip('\n')) -@pytest.fixture -def sample_module(tmp_path: Path) -> str: - """ - Write a module and put it in :py:data:`sys.path`. When we're done, - reset :py:data:`sys.path` and `sys.modules`. - """ - def gen_names() -> Generator[str, None, None]: - while True: - yield 'my_module_' + str(uuid4()).replace('-', '_') +def gen_names(name) -> Generator[str, None, None]: + while True: + yield '_'.join([name, *str(uuid4()).split('-')]) + +@pytest.fixture +def preserve_sys_state() -> None: old_path = sys.path.copy() old_modules = sys.modules.copy() - module_name = next(name for name in gen_names() if name not in old_modules) + try: + yield + finally: + sys.path.clear() + sys.path[:] = old_path + sys.modules.clear() + sys.modules.update(old_modules) + + +@pytest.fixture +def sample_package(preserve_sys_state: None, tmp_path: Path) -> str: + """ + Write a normal package and put it in :py:data:`sys.path`. When + we're done, reset :py:data:`sys.path` and `sys.modules`. + """ + module_name = next(name for name in gen_names('my_sample_pkg') + if name not in sys.modules) new_path = tmp_path / '_modules' write(new_path / module_name / '__init__.py') write(new_path / module_name / 'foo' / '__init__.py') @@ -63,16 +77,28 @@ def gen_names() -> Generator[str, None, None]: raise AssertionError """) write(new_path / module_name / 'foobar.py') - try: + # Cleanup managed with `preserve_sys_state()` + sys.path.insert(0, str(new_path)) + yield module_name + + +@pytest.fixture +def sample_namespace_package( + preserve_sys_state: None, + tmp_path_factory: pytest.TempPathFactory) -> str: + """ + Write a namespace package and put it in :py:data:`sys.path`. When + we're done, reset :py:data:`sys.path` and `sys.modules`. + """ + module_name = next(name for name in gen_names('my_sample_namespace_pkg') + if name not in sys.modules) + new_paths = [tmp_path_factory.mktemp('_modules-', numbered=True) + for _ in range(3)] + for submod, new_path in zip(['one', 'two', 'three'], new_paths): + write(new_path / module_name / (submod + '.py')) + # Cleanup managed with `preserve_sys_state()` sys.path.insert(0, str(new_path)) - yield module_name - finally: - sys.path.clear() - sys.path[:] = old_path - assert str(new_path) not in sys.path - sys.modules.clear() - sys.modules.update(old_modules) - assert module_name not in sys.modules + yield module_name @pytest.mark.parametrize( @@ -85,11 +111,11 @@ def test_write_eager_import_module_wrong_adder( :py:meth:`~.write_eager_import_module()`. """ with pytest.raises(xc): - write_module(['foo'], adder=adder) + write_eager_import_module(['foo'], adder=adder) @pytest.mark.skipif(not HAS_FLAKE8, reason='no `flake8`') -def test_written_module_pep8_compliance(sample_module: str): +def test_written_module_pep8_compliance(sample_package: str): """ Test that the module written by :py:meth:`~.write_eager_import_module()` passes linting by @@ -98,8 +124,9 @@ def test_written_module_pep8_compliance(sample_module: str): with TemporaryDirectory() as tmpdir: module = Path(tmpdir) / 'module.py' with module.open(mode='w') as fobj: - write_module([sample_module + '.foobar'], - recurse=[sample_module + '.foo'], stream=fobj) + write_eager_import_module( + [sample_package + '.foobar'], + recurse=[sample_package + '.foo'], stream=fobj) print(module.read_text()) (subprocess .run([sys.executable, '-m', 'flake8', @@ -133,7 +160,7 @@ def test_written_module_pep8_compliance(sample_module: str): # error (['__MODULE__', '__MODULE__.foo.baz'], False, [], AssertionError)]) def test_written_module_error_handling( - sample_module: str, + sample_package: str, dotted_paths: Collection[str], recurse: Union[Collection[str], bool], warnings: Sequence[Collection[str]], @@ -143,7 +170,7 @@ def test_written_module_error_handling( :py:meth:`~.write_eager_import_module()` gracefully handles errors for implicitly included modules. """ - replace = methodcaller('replace', '__MODULE__', sample_module) + replace = methodcaller('replace', '__MODULE__', sample_package) dotted_paths = [replace(target) for target in dotted_paths] if recurse not in (True, False): recurse = [replace(target) for target in recurse] @@ -157,7 +184,8 @@ def test_written_module_error_handling( # warnings at module-generation time and execution time captured_warnings = enter(catch_warnings(record=True)) with module.open(mode='w') as fobj: - write_module(dotted_paths, recurse=recurse, stream=fobj) + write_eager_import_module( + dotted_paths, recurse=recurse, stream=fobj) print(module.read_text()) if error is not None: enter(pytest.raises(error)) @@ -169,3 +197,34 @@ def test_written_module_error_handling( for warning, fragments in zip(captured_warnings, warnings): for fragment in fragments: assert fragment in str(warning.message) + + +def test_split_dotted_path_staticity() -> None: + """ + Test `split_dotted_path()` with different values for `static`. + """ + split = partial(split_dotted_path, 'os.path.abspath') + # Static analysis has no idea of `os.path` members since `os.path` + # is dynamically imported from e.g. `posixpath` or `ntpath` + assert split(static=True) == ('os', 'path.abspath') + # The import system knows of the already-imported module `os.path` + assert split(static=False) == ('os.path', 'abspath') + + +def test_resolve_profiling_targets_staticity( + sample_namespace_package: str) -> None: + """ + Test subpackage/-module discovery with `resolve_profiling_targets()` + with different values for `static`. + """ + all_targets = ({f'{sample_namespace_package}.{submod}' + for submod in ['one', 'two', 'three']} + | {sample_namespace_package}) + # Static analysis can't handle namespace packages + resolve = partial( + resolve_profiling_targets, [sample_namespace_package], recurse=True) + static_result = resolve(static=True) + assert set(static_result.targets) < all_targets, static_result + # The import system successfully retrieves all submodules + dyn_result = resolve(static=False) + assert set(dyn_result.targets) == all_targets, dyn_result From 7288e1a55e145a28c3a2eaaaa55ec000db24ce2f Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 15 Jun 2025 10:50:49 +0200 Subject: [PATCH 59/70] Default to import-based pre-imports in `kernprof` kernprof.py::main(), _write_preimports() Now choosing whether to use static-only analysis when writing the pre-import module by `line_profiler._diagnostics.STATIC_ANALYSIS` line_profiler/_diagnostics.py::STATIC_ANALYSIS New "dev-mode" variable set by the environment variable `${LINE_PROFILER_STATIC_ANALYSIS}` --- kernprof.py | 5 +++-- line_profiler/_diagnostics.py | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/kernprof.py b/kernprof.py index 777949be..ac5af406 100755 --- a/kernprof.py +++ b/kernprof.py @@ -661,8 +661,9 @@ def no_op(*_, **__) -> None: # Hand off to the dummy parser if necessary to generate the help # text options = SimpleNamespace(**vars(real_parser.parse_args(args))) - # TODO: make a flag later where appropriate + # TODO: make flags later where appropriate options.dryrun = diagnostics.NO_EXEC + options.static = diagnostics.STATIC_ANALYSIS options.rm = no_op if diagnostics.KEEP_TEMPDIRS else _remove if help_parser and getattr(options, 'help', False): help_parser.print_help() @@ -860,7 +861,7 @@ def _write_preimports(prof, options, exclude): suffix='.py') write_module = functools.partial( write_eager_import_module, filtered_targets, - recurse=recurse_targets) + recurse=recurse_targets, static=options.static) temp_file = open(temp_mod_path, mode='w') if options.debug: with StringIO() as sio: diff --git a/line_profiler/_diagnostics.py b/line_profiler/_diagnostics.py index d368a41b..afaecfe3 100644 --- a/line_profiler/_diagnostics.py +++ b/line_profiler/_diagnostics.py @@ -22,5 +22,6 @@ def _boolean_environ(key): DEBUG = _boolean_environ('LINE_PROFILER_DEBUG') NO_EXEC = _boolean_environ('LINE_PROFILER_NO_EXEC') KEEP_TEMPDIRS = _boolean_environ('LINE_PROFILER_KEEP_TEMPDIRS') +STATIC_ANALYSIS = _boolean_environ('LINE_PROFILER_STATIC_ANALYSIS') log = _logger.Logger('line_profiler') From 7f23e239cd0f36c12782a03d7e3cd18d4822ef64 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 15 Jun 2025 23:49:47 +0200 Subject: [PATCH 60/70] Logging function adds line_profiler/line_profiler.py[i] LineProfiler.add_callable() - New argument `name` for referring to the added object in log messages - Now emitting a log message for each of the underlying functions LineProfiler.add_class(), .add_module() Now emitting a log message if any member in the class/module has been added --- line_profiler/line_profiler.py | 79 +++++++++++++++++++++++++++------ line_profiler/line_profiler.pyi | 5 ++- 2 files changed, 68 insertions(+), 16 deletions(-) diff --git a/line_profiler/line_profiler.py b/line_profiler/line_profiler.py index e2c209a3..5c7c40ea 100755 --- a/line_profiler/line_profiler.py +++ b/line_profiler/line_profiler.py @@ -13,6 +13,7 @@ import tempfile import types from argparse import ArgumentError, ArgumentParser +from datetime import datetime try: from ._line_profiler import LineProfiler as CLineProfiler @@ -21,6 +22,7 @@ 'The line_profiler._line_profiler c-extension is not importable. ' f'Has it been compiled? Underlying error is ex={ex!r}' ) +from . import _diagnostics as diagnostics from .profiler_mixin import ByCountProfilerMixin, is_c_level_callable from .scoping_policy import ScopingPolicy @@ -89,7 +91,7 @@ def wrap_callable(self, func): return func return super().wrap_callable(func) - def add_callable(self, func, guard=None): + def add_callable(self, func, guard=None, name=None): """ Register a function, method, :py:class:`property`, :py:func:`~functools.partial` object, etc. with the underlying @@ -103,6 +105,8 @@ def add_callable(self, func, guard=None): and returns true(-y) if it *should not* be passed to :py:meth:`.add_function()`. Defaults to checking whether the function is already a profiling wrapper. + name (Optional[str]) + Optional name for ``func``, to be used in log messages. Returns: 1 if any function is added to the profiler, 0 otherwise. @@ -115,6 +119,7 @@ def add_callable(self, func, guard=None): guard = self._already_a_wrapper nadded = 0 + func_repr = self._repr_for_log(func, name) for impl in self.get_underlying_functions(func): info, wrapped_by_this_prof = self._get_wrapper_info(impl) if wrapped_by_this_prof if guard is None else guard(impl): @@ -125,9 +130,39 @@ def add_callable(self, func, guard=None): impl = info.func self.add_function(impl) nadded += 1 + if impl is func: + self._debug(f'added {func_repr}') + else: + self._debug(f'added {func_repr} -> {self._repr_for_log(impl)}') return 1 if nadded else 0 + @staticmethod + def _repr_for_log(obj, name=None): + try: + real_name = '{0.__module__}.{0.__qualname__}'.format(obj) + except AttributeError: + try: + real_name = obj.__name__ + except AttributeError: + real_name = '???' + return '{} `{}{}` {}@ {:#x}'.format( + type(obj).__name__, + real_name, + '()' if callable(obj) and not isinstance(obj, type) else '', + f'(=`{name}`) ' if name and name != real_name else '', + id(obj)) + + def _debug(self, msg): + self_repr = f'{type(self).__name__} @ {id(self):#x}' + logger = diagnostics.log + if logger.backend == 'print': + now = datetime.now().isoformat(sep=' ', timespec='seconds') + msg = f'[{self_repr} {now}] {msg}' + else: + msg = f'{self_repr}: {msg}' + logger.debug(msg) + def dump_stats(self, filename): """ Dump a representation of the data to a file as a pickled :py:class:`~.LineStats` object from :py:meth:`~.get_stats()`. @@ -151,7 +186,8 @@ def _add_namespace( func_scoping_policy=ScopingPolicy.NONE, class_scoping_policy=ScopingPolicy.NONE, module_scoping_policy=ScopingPolicy.NONE, - wrap=False): + wrap=False, + name=None): def func_guard(func): return self._already_a_wrapper(func) or not func_check(func) @@ -170,29 +206,44 @@ def func_guard(func): cls_check = class_scoping_policy.get_filter(namespace, 'class') mod_check = module_scoping_policy.get_filter(namespace, 'module') + # Logging stuff + if not name: + try: # Class + name = '{0.__module__}.{0.__qualname__}'.format(namespace) + except AttributeError: # Module + name = namespace.__name__ + for attr, value in vars(namespace).items(): if id(value) in seen: continue seen.add(id(value)) if isinstance(value, type): - if cls_check(value) and add_namespace(value): - count += 1 - continue + if not (cls_check(value) + and add_namespace(value, name=f'{name}.{attr}')): + continue elif isinstance(value, types.ModuleType): - if mod_check(value) and add_namespace(value): - count += 1 - continue - try: - if not self.add_callable(value, guard=func_guard): + if not (mod_check(value) + and add_namespace(value, name=f'{name}.{attr}')): continue - except TypeError: # Not a callable (wrapper) - continue - if wrap: - members_to_wrap[attr] = value + else: + try: + if not self.add_callable( + value, guard=func_guard, name=f'{name}.{attr}'): + continue + except TypeError: # Not a callable (wrapper) + continue + if wrap: + members_to_wrap[attr] = value count += 1 if wrap and members_to_wrap: self._wrap_namespace_members(namespace, members_to_wrap, warning_stack_level=3) + if count: + self._debug( + 'added {} member{} in {}'.format( + count, + '' if count == 1 else 's', + self._repr_for_log(namespace, name))) return count def add_class(self, cls, *, scoping_policy=None, wrap=False): diff --git a/line_profiler/line_profiler.pyi b/line_profiler/line_profiler.pyi index f794e50f..89a0fa3a 100644 --- a/line_profiler/line_profiler.pyi +++ b/line_profiler/line_profiler.pyi @@ -36,8 +36,9 @@ class LineProfiler(CLineProfiler, ByCountProfilerMixin): ... def add_callable( - self, func, guard: (Callable[[FunctionType], bool] - | None) = None) -> Literal[0, 1]: + self, func, + guard: Callable[[FunctionType], bool] | None = None, + name: str | None = None) -> Literal[0, 1]: ... def dump_stats(self, filename) -> None: From e842d23697668ebeeeac06059d7762cdbd27b60e Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 16 Jun 2025 00:42:36 +0200 Subject: [PATCH 61/70] Better (non-)handling of Cython functions line_profiler/profiler_mixin.py[i] is_cython_callable() - Now a separate function - Now checking the name of the object's type instead of doing an instance check, so that we retain compatibility with extension modules built with different Cython versions is_c_level_callable() Now calls `is_cython_callable()`, instead of hard-coding `type(line_profiler._line_profiler.label)` into `C_LEVEL_CALLABLE_TYPES` --- line_profiler/profiler_mixin.py | 17 +++++++++++++---- line_profiler/profiler_mixin.pyi | 4 ++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/line_profiler/profiler_mixin.py b/line_profiler/profiler_mixin.py index ef803fb5..ab4cc59a 100644 --- a/line_profiler/profiler_mixin.py +++ b/line_profiler/profiler_mixin.py @@ -2,7 +2,6 @@ import inspect import types from warnings import warn -from ._line_profiler import label from .scoping_policy import ScopingPolicy @@ -18,8 +17,7 @@ types.ClassMethodDescriptorType, types.MethodDescriptorType, types.MethodWrapperType, - types.WrapperDescriptorType, - type(label)) + types.WrapperDescriptorType) def is_c_level_callable(func): @@ -29,7 +27,18 @@ def is_c_level_callable(func): Whether a callable is defined at the C(-ython) level (and is thus non-profilable). """ - return isinstance(func, C_LEVEL_CALLABLE_TYPES) + return isinstance(func, C_LEVEL_CALLABLE_TYPES) or is_cython_callable(func) + + +def is_cython_callable(func): + if not callable(func): + return False + # Note: don't directly check against a Cython function type, since + # said type depends on the Cython version used for building the + # Cython code; + # just check for what is common between Cython versions + return (type(func).__name__ + in ('cython_function_or_method', 'fused_cython_function')) def is_classmethod(f): diff --git a/line_profiler/profiler_mixin.pyi b/line_profiler/profiler_mixin.pyi index 3085a4a7..166cc39f 100644 --- a/line_profiler/profiler_mixin.pyi +++ b/line_profiler/profiler_mixin.pyi @@ -128,6 +128,10 @@ def is_c_level_callable(func: Any) -> TypeIs[CLevelCallable]: ... +def is_cython_callable(func: Any) -> TypeIs[CythonCallable]: + ... + + def is_classmethod(f: Any) -> TypeIs[classmethod]: ... From 447ab0a6229ff4fae1b2406043cf93da9a513fa3 Mon Sep 17 00:00:00 2001 From: joncrall Date: Sat, 5 Jul 2025 15:00:02 -0400 Subject: [PATCH 62/70] remove code diagnostics --- kernprof.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/kernprof.py b/kernprof.py index ac5af406..d09671ea 100755 --- a/kernprof.py +++ b/kernprof.py @@ -704,7 +704,6 @@ def no_op(*_, **__) -> None: else: options.message = logger.info options.diagnostics = print_diagnostics - options.code_diagnostics = print_code_block_diagnostics if options.rich: try: import rich # noqa: F401 @@ -712,7 +711,6 @@ def no_op(*_, **__) -> None: options.rich = False options.diagnostics('`rich` not installed, unsetting --rich') - options.code_diagnostics('Parser output:', pformat(options)) if module is not None: options.diagnostics('Profiling module:', module) elif tempfile_source_and_content: @@ -784,8 +782,7 @@ def _write_tempfile(source, content, options): fname = os.path.join(options.tmpdir, file_prefix + '.py') with open(fname, mode='w') as fobj: print(content, file=fobj) - options.code_diagnostics(f'Wrote temporary script file to {fname!r}:', - content) + options.diagnostics(f'Wrote temporary script file to {fname!r}:') options.script = fname # Add the tempfile to `--prof-mod` if options.prof_mod: @@ -869,10 +866,9 @@ def _write_preimports(prof, options, exclude): code = sio.getvalue() with temp_file as fobj: print(code, file=fobj) - options.code_diagnostics( + options.diagnostics( 'Wrote temporary module for pre-imports ' - f'to {temp_mod_path!r}:', - code) + f'to {temp_mod_path!r}:') else: with temp_file as fobj: write_module(stream=fobj) @@ -923,7 +919,7 @@ def call_with_diagnostics(func, *args, **kwargs): func_repr, indent(all_args_repr, ' ')) else: call = func_repr + '()' - options.code_diagnostics('Calling:', call) + options.diagnostics('Calling:', call) if options.dryrun: return return func(*args, **kwargs) From d942680c12e36ce3025ae21f3ddd7dfa597df443 Mon Sep 17 00:00:00 2001 From: joncrall Date: Sat, 5 Jul 2025 15:00:38 -0400 Subject: [PATCH 63/70] remove code diagnostics --- kernprof.py | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/kernprof.py b/kernprof.py index d09671ea..3f59e334 100755 --- a/kernprof.py +++ b/kernprof.py @@ -526,31 +526,6 @@ def print_diagnostics(*args, **kwargs): printer(*args, **kwargs) logger.debug(sio.getvalue()) - def print_code_block_diagnostics( - header, code, *, line_numbers=True, **kwargs): - if options.rich and simple_logging: - from rich.syntax import Syntax - - code_repr = Syntax(code, 'python', line_numbers=line_numbers) - elif line_numbers: - nlines = code.count('\n') + 1 - line_number_width = len(str(nlines)) + 2 - code_repr = ''.join( - f'{n:>{line_number_width}} {line}' - for n, line in zip(range(1, nlines + 1), - code.splitlines(keepends=True))) - else: - code_repr = code - kwargs['sep'] = '\n' - - # Insert additional space - if not header.endswith('\n'): - header += '\n' - args = [header, code_repr] - if not code.endswith('\n'): - args.append('') - print_diagnostics(*args, **kwargs) - def no_op(*_, **__) -> None: pass From 1b88d97156d17ce429d3d1bf65e1c51082d03308 Mon Sep 17 00:00:00 2001 From: joncrall Date: Sat, 5 Jul 2025 15:43:11 -0400 Subject: [PATCH 64/70] Simplified logger usage --- kernprof.py | 94 +++++++++++++++-------------------- line_profiler/_diagnostics.py | 2 +- tests/test_kernprof.py | 17 ++++--- 3 files changed, 51 insertions(+), 62 deletions(-) diff --git a/kernprof.py b/kernprof.py index 3f59e334..900f7207 100755 --- a/kernprof.py +++ b/kernprof.py @@ -142,7 +142,6 @@ def main(): import time import warnings from argparse import ArgumentError, ArgumentParser -from datetime import datetime from io import StringIO from operator import methodcaller from runpy import run_module @@ -508,30 +507,12 @@ def positive_float(value): raise ArgumentError return val - def print_diagnostics(*args, **kwargs): - with StringIO() as sio: - if options.rich and simple_logging: - from rich.console import Console - from rich.markup import escape - - printer = Console(file=sio, force_terminal=True).print - else: - escape = str - printer = functools.partial(print, file=sio) - - if args and simple_logging: - now = datetime.now().isoformat(sep=' ', timespec='seconds') - args = ['{} {}'.format(escape(f'[kernprof {now}]'), args[0]), - *args[1:]] - printer(*args, **kwargs) - logger.debug(sio.getvalue()) - def no_op(*_, **__) -> None: pass - create_parser = functools.partial( - ArgumentParser, - description='Run and profile a python script.') + parser_kwargs = { + 'description': 'Run and profile a python script.', + } if args is None: args = sys.argv[1:] @@ -552,7 +533,7 @@ def no_op(*_, **__) -> None: module, literal_code = None, thing if module is literal_code is None: # Normal execution - real_parser, = parsers = [create_parser()] + real_parser, = parsers = [ArgumentParser(**parser_kwargs)] help_parser = None else: # We've already consumed the `-m `, so we need a dummy @@ -560,9 +541,9 @@ def no_op(*_, **__) -> None: # but the real parser should not consume the `options.script` # positional arg, and it it got the `--help` option, it should # hand off the the dummy parser - real_parser = create_parser(add_help=False) + real_parser = ArgumentParser(add_help=False, **parser_kwargs) real_parser.add_argument('-h', '--help', action='store_true') - help_parser = create_parser() + help_parser = ArgumentParser(**parser_kwargs) parsers = [real_parser, help_parser] for parser in parsers: parser.add_argument('-V', '--version', action='version', version=__version__) @@ -639,7 +620,6 @@ def no_op(*_, **__) -> None: # TODO: make flags later where appropriate options.dryrun = diagnostics.NO_EXEC options.static = diagnostics.STATIC_ANALYSIS - options.rm = no_op if diagnostics.KEEP_TEMPDIRS else _remove if help_parser and getattr(options, 'help', False): help_parser.print_help() exit() @@ -664,35 +644,34 @@ def no_op(*_, **__) -> None: options.debug = (diagnostics.DEBUG or options.verbose >= DIAGNOSITICS_VERBOSITY) logger_kwargs = {'name': 'kernprof'} + logger_kwargs['backend'] = 'auto' if options.debug: + # Debugging forces the stdlib logger logger_kwargs['verbose'] = 2 + logger_kwargs['backend'] = 'stdlib' elif options.verbose > -1: logger_kwargs['verbose'] = 1 else: logger_kwargs['verbose'] = 0 - # Also consume log items written from other `line_profiler` - # components - logger = diagnostics.log = Logger(**logger_kwargs) - simple_logging = logger.backend == 'print' - if options.debug: - options.message = print_diagnostics - else: - options.message = logger.info - options.diagnostics = print_diagnostics + logger_kwargs['stream'] = { + 'format': '[%(name)s %(asctime)s %(levelname)s] %(message)s', + } + # Reinitialize the diagnostic logs, we are very likely the main script. + diagnostics.log = Logger(**logger_kwargs) if options.rich: try: import rich # noqa: F401 except ImportError: options.rich = False - options.diagnostics('`rich` not installed, unsetting --rich') + diagnostics.log.debug('`rich` not installed, unsetting --rich') if module is not None: - options.diagnostics('Profiling module:', module) + diagnostics.log.debug(f'Profiling module: {module}') elif tempfile_source_and_content: - options.diagnostics('Profiling script read from', - tempfile_source_and_content[0]) + diagnostics.log.debug( + f'Profiling script read from: {tempfile_source_and_content[0]}') else: - options.diagnostics('Profiling script:', options.script) + diagnostics.log.debug(f'Profiling script: {options.script}') with contextlib.ExitStack() as stack: enter = stack.enter_context @@ -705,9 +684,12 @@ def no_op(*_, **__) -> None: # manage a tempdir to ensure that files exist at # traceback-formatting time if needs be options.tmpdir = tmpdir = tempfile.mkdtemp() - cleanup = functools.partial( - options.rm, tmpdir, recursive=True, missing_ok=True, - ) + if diagnostics.KEEP_TEMPDIRS: + cleanup = no_op + else: + cleanup = functools.partial( + _remove, tmpdir, recursive=True, missing_ok=True, + ) if tempfile_source_and_content: try: _write_tempfile(*tempfile_source_and_content, options) @@ -757,7 +739,7 @@ def _write_tempfile(source, content, options): fname = os.path.join(options.tmpdir, file_prefix + '.py') with open(fname, mode='w') as fobj: print(content, file=fobj) - options.diagnostics(f'Wrote temporary script file to {fname!r}:') + diagnostics.log.debug(f'Wrote temporary script file to {fname!r}:') options.script = fname # Add the tempfile to `--prof-mod` if options.prof_mod: @@ -771,7 +753,7 @@ def _write_tempfile(source, content, options): options.outfile = _touch_tempfile(dir=os.curdir, prefix=file_prefix + '-', suffix='.' + extension) - options.diagnostics( + diagnostics.log.debug( f'Using default output destination {options.outfile!r}') @@ -821,6 +803,7 @@ def _write_preimports(prof, options, exclude): '' if len(invalid_targets) == 1 else 's', invalid_targets)) warnings.warn(msg) + diagnostics.log.warn(msg) if not (filtered_targets or recurse_targets): return # We could've done everything in-memory with `io.StringIO` and @@ -841,7 +824,7 @@ def _write_preimports(prof, options, exclude): code = sio.getvalue() with temp_file as fobj: print(code, file=fobj) - options.diagnostics( + diagnostics.log.debug( 'Wrote temporary module for pre-imports ' f'to {temp_mod_path!r}:') else: @@ -851,7 +834,10 @@ def _write_preimports(prof, options, exclude): ns = {} # Use a fresh namespace execfile(temp_mod_path, ns, ns) # Delete the tempfile ASAP if its execution succeeded - options.rm(temp_mod_path) + if diagnostics.KEEP_TEMPDIRS: + diagnostics.log.debug('Keep temporary preimport path: {temp_mod_path}') + else: + _remove(temp_mod_path) def _remove(path, *, recursive=False, missing_ok=False): @@ -894,7 +880,7 @@ def call_with_diagnostics(func, *args, **kwargs): func_repr, indent(all_args_repr, ' ')) else: call = func_repr + '()' - options.diagnostics('Calling:', call) + diagnostics.log.debug(f'Calling: {call}') if options.dryrun: return return func(*args, **kwargs) @@ -927,7 +913,7 @@ def dump_filtered_stats(prof, filename): if not options.outfile: extension = 'lprof' if options.line_by_line else 'prof' options.outfile = f'{os.path.basename(options.script)}.{extension}' - options.diagnostics( + diagnostics.log.debug( f'Using default output destination {options.outfile!r}') sys.argv = [options.script] + options.args @@ -945,7 +931,7 @@ def dump_filtered_stats(prof, filename): # kernprof.py's. sys.path.insert(0, os.path.dirname(setup_file)) ns = {'__file__': setup_file, '__name__': '__main__'} - options.diagnostics( + diagnostics.log.debug( f'Executing file {setup_file!r} as pre-profiling setup') if not options.dryrun: execfile(setup_file, ns, ns) @@ -1041,7 +1027,7 @@ def dump_filtered_stats(prof, filename): rt.stop() if not options.dryrun: dump_filtered_stats(prof, options.outfile) - options.message( + diagnostics.log.info( ('Profile results would have been written to ' if options.dryrun else 'Wrote profile results ') @@ -1060,9 +1046,9 @@ def dump_filtered_stats(prof, filename): show_mod = 'pstats' else: show_mod = 'line_profiler -rmt' - options.message('Inspect results with:\n' - f'{quote(py_exe)} -m {show_mod} ' - f'{quote(options.outfile)}') + diagnostics.log.info('Inspect results with:\n' + f'{quote(py_exe)} -m {show_mod} ' + f'{quote(options.outfile)}') # Fully disable the profiler for _ in range(prof.enable_count): prof.disable_by_count() diff --git a/line_profiler/_diagnostics.py b/line_profiler/_diagnostics.py index afaecfe3..8ef2cc68 100644 --- a/line_profiler/_diagnostics.py +++ b/line_profiler/_diagnostics.py @@ -24,4 +24,4 @@ def _boolean_environ(key): KEEP_TEMPDIRS = _boolean_environ('LINE_PROFILER_KEEP_TEMPDIRS') STATIC_ANALYSIS = _boolean_environ('LINE_PROFILER_STATIC_ANALYSIS') -log = _logger.Logger('line_profiler') +log = _logger.Logger('line_profiler', backend='auto') diff --git a/tests/test_kernprof.py b/tests/test_kernprof.py index ed3621de..de9bb871 100644 --- a/tests/test_kernprof.py +++ b/tests/test_kernprof.py @@ -195,10 +195,8 @@ def main(): ('--view', # Verbosity level 1 = `--view` {'^Output to stdout': True, r"^Wrote .* '.*script\.py\.lprof'": True, - r'Parser output:''(?:\n)+'r'.*namespace\((?:.+,\n)*.*\)': False, r'^Inspect results with:''\n' r'python -m line_profiler .*script\.py\.lprof': False, - '^ *[0-9]+ *import zipfile': False, r'line_profiler\.autoprofile\.autoprofile' r'\.run\(\n(?:.+,\n)*.*\)': False, r'^\[kernprof .*\]': False, @@ -207,8 +205,6 @@ def main(): ('-vv', # Verbosity level 2, show diagnostics {'^Output to stdout': True, r"^\[kernprof .*\] Wrote .* '.*script\.py\.lprof'": True, - r'Parser output:''(?:\n)+'r'.*namespace\((?:.+,\n)*.*\)': True, - '^ *[0-9]+ *import zipfile': True, r'Inspect results with:''\n' r'python -m line_profiler .*script\.py\.lprof': False, r'line_profiler\.autoprofile\.autoprofile' @@ -251,7 +247,7 @@ def main(): enter(ub.ChDir(tmpdir)) proc = ub.cmd(['kernprof', '-l', # Add an eager pre-import target - '-pscript.py', '-pzipfile', '-z', + '-p', 'script.py', '-p', 'zipfile', '-z', *shlex.split(flags), 'script.py']) proc.check_returncode() print(proc.stdout) @@ -261,8 +257,15 @@ def main(): assert not stream continue for pattern, expect_match in expected_outputs.items(): - assert bool(re.search(pattern, stream, - flags=re.MULTILINE)) == expect_match + found = re.search(pattern, stream, flags=re.MULTILINE) + if not bool(found) == expect_match: + msg = ub.paragraph( + f''' + Searching for pattern: {pattern!r} in output. + Did we expect a match? {expect_match!r}. + Did we get a match? {bool(found)!r}. + ''') + raise AssertionError(msg) def test_kernprof_eager_preimport_bad_module(): From 67630312f1b14c02db0bff75d32d4f29850b87d0 Mon Sep 17 00:00:00 2001 From: joncrall Date: Sat, 5 Jul 2025 15:52:30 -0400 Subject: [PATCH 65/70] refactor functools.partial in write_module --- kernprof.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/kernprof.py b/kernprof.py index 900f7207..8656db01 100755 --- a/kernprof.py +++ b/kernprof.py @@ -814,13 +814,15 @@ def _write_preimports(prof, options, exclude): temp_mod_path = _touch_tempfile(dir=options.tmpdir, prefix='kernprof-eager-preimports-', suffix='.py') - write_module = functools.partial( - write_eager_import_module, filtered_targets, - recurse=recurse_targets, static=options.static) + write_module_kwargs = { + 'dotted_paths': filtered_targets, + 'recurse': recurse_targets, + 'static': options.static, + } temp_file = open(temp_mod_path, mode='w') if options.debug: with StringIO() as sio: - write_module(stream=sio) + write_eager_import_module(stream=sio, **write_module_kwargs) code = sio.getvalue() with temp_file as fobj: print(code, file=fobj) @@ -829,7 +831,7 @@ def _write_preimports(prof, options, exclude): f'to {temp_mod_path!r}:') else: with temp_file as fobj: - write_module(stream=fobj) + write_eager_import_module(stream=fobj, **write_module_kwargs) if not options.dryrun: ns = {} # Use a fresh namespace execfile(temp_mod_path, ns, ns) From e059764e9a8689f2be854795e96d81cffe6c2aa9 Mon Sep 17 00:00:00 2001 From: joncrall Date: Sat, 5 Jul 2025 15:56:55 -0400 Subject: [PATCH 66/70] Refactor dump_filtered_stats --- kernprof.py | 59 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 33 insertions(+), 26 deletions(-) diff --git a/kernprof.py b/kernprof.py index 8656db01..62cc41ee 100755 --- a/kernprof.py +++ b/kernprof.py @@ -853,6 +853,38 @@ def _remove(path, *, recursive=False, missing_ok=False): path.unlink(missing_ok=missing_ok) +def _dump_filtered_stats(tmpdir, prof, filename): + import os + import pickle + + # Build list of known temp file paths + tempfile_paths = [ + os.path.join(dirpath, fname) + for dirpath, _, fnames in os.walk(tmpdir) + for fname in fnames + ] + + if not tempfile_paths: + prof.dump_stats(filename) + return + + # Filter the filenames to remove data from tempfiles, which will + # have been deleted by the time the results are viewed in a + # separate process + stats = prof.get_stats() + timings = stats.timings + for key in set(timings): + fname = key[0] + try: + if any(os.path.samefile(fname, tmp) for tmp in tempfile_paths): + del timings[key] + except OSError: + del timings[key] + + with open(filename, 'wb') as f: + pickle.dump(stats, f, protocol=pickle.HIGHEST_PROTOCOL) + + def _main(options, module=False): """ Called by :py:func:`main()` for the actual execution and profiling @@ -887,31 +919,6 @@ def call_with_diagnostics(func, *args, **kwargs): return return func(*args, **kwargs) - def dump_filtered_stats(prof, filename): - import pickle - - tempfile_checks = {functools.partial(os.path.samefile, - os.path.join(dirname, fname)) - for dirname, _, fnames in os.walk(options.tmpdir) - for fname in fnames} - if not tempfile_checks: - return prof.dump_stats(filename) - # Filter the filenames to remove data from tempfiles, which will - # have been deleted by the time the results are viewed in a - # separate process - stats = prof.get_stats() - timings = stats.timings - for key in set(timings): - fname, *_ = key - try: - del_key = any(check(fname) for check in tempfile_checks) - except OSError: - del_key = True - if del_key: - del timings[key] - with open(filename, mode='wb') as fobj: - pickle.dump(stats, fobj, protocol=pickle.HIGHEST_PROTOCOL) - if not options.outfile: extension = 'lprof' if options.line_by_line else 'prof' options.outfile = f'{os.path.basename(options.script)}.{extension}' @@ -1028,7 +1035,7 @@ def dump_filtered_stats(prof, filename): if use_timer: rt.stop() if not options.dryrun: - dump_filtered_stats(prof, options.outfile) + _dump_filtered_stats(options.tmpdir, prof, options.outfile) diagnostics.log.info( ('Profile results would have been written to ' if options.dryrun else From 161ddf84e84a638442e9cf8f9f3a5460ca802b14 Mon Sep 17 00:00:00 2001 From: joncrall Date: Sat, 5 Jul 2025 16:08:39 -0400 Subject: [PATCH 67/70] Simplify _write_preimports --- kernprof.py | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/kernprof.py b/kernprof.py index 62cc41ee..33b8d377 100755 --- a/kernprof.py +++ b/kernprof.py @@ -757,17 +757,12 @@ def _write_tempfile(source, content, options): f'Using default output destination {options.outfile!r}') -def _write_preimports(prof, options, exclude): +def _gather_preimport_targets(options, exclude): """ - Called by :py:func:`main()` to handle eager pre-imports; - not to be invoked on its own. + Used in _write_preimports """ - from line_profiler.autoprofile.eager_preimports import ( - is_dotted_path, write_eager_import_module) from line_profiler.autoprofile.util_static import modpath_to_modname - from line_profiler.autoprofile.autoprofile import ( - _extend_line_profiler_for_profiling_imports as upgrade_profiler) - + from line_profiler.autoprofile.eager_preimports import is_dotted_path filtered_targets = [] recurse_targets = [] invalid_targets = [] @@ -791,10 +786,9 @@ def _write_preimports(prof, options, exclude): continue if modname.endswith('.__init__'): modname = modname.rpartition('.')[0] - targets = filtered_targets + filtered_targets.append(modname) else: - targets = recurse_targets - targets.append(modname) + recurse_targets.append(modname) if invalid_targets: invalid_targets = sorted(set(invalid_targets)) msg = ('{} profile-on-import target{} cannot be converted to ' @@ -804,11 +798,24 @@ def _write_preimports(prof, options, exclude): invalid_targets)) warnings.warn(msg) diagnostics.log.warn(msg) + + return filtered_targets, recurse_targets + + +def _write_preimports(prof, options, exclude): + """ + Called by :py:func:`main()` to handle eager pre-imports; + not to be invoked on its own. + """ + from line_profiler.autoprofile.eager_preimports import write_eager_import_module + from line_profiler.autoprofile.autoprofile import ( + _extend_line_profiler_for_profiling_imports as upgrade_profiler) + + filtered_targets, recurse_targets = _gather_preimport_targets(options, exclude) if not (filtered_targets or recurse_targets): return - # We could've done everything in-memory with `io.StringIO` and - # `exec()`, but that results in indecipherable tracebacks should - # anything goes wrong; + # We could've done everything in-memory with `io.StringIO` and `exec()`, + # but that results in indecipherable tracebacks should anything goes wrong; # so we write to a tempfile and `execfile()` it upgrade_profiler(prof) temp_mod_path = _touch_tempfile(dir=options.tmpdir, @@ -836,9 +843,7 @@ def _write_preimports(prof, options, exclude): ns = {} # Use a fresh namespace execfile(temp_mod_path, ns, ns) # Delete the tempfile ASAP if its execution succeeded - if diagnostics.KEEP_TEMPDIRS: - diagnostics.log.debug('Keep temporary preimport path: {temp_mod_path}') - else: + if not diagnostics.KEEP_TEMPDIRS: _remove(temp_mod_path) From cad8998bc3f3355371885186be7cded4015d4326 Mon Sep 17 00:00:00 2001 From: joncrall Date: Sat, 5 Jul 2025 16:08:53 -0400 Subject: [PATCH 68/70] Move _call_with_diagnostics to global scope --- kernprof.py | 69 ++++++++++++++++++++++++++++------------------------- 1 file changed, 37 insertions(+), 32 deletions(-) diff --git a/kernprof.py b/kernprof.py index 33b8d377..97e38b05 100755 --- a/kernprof.py +++ b/kernprof.py @@ -890,39 +890,41 @@ def _dump_filtered_stats(tmpdir, prof, filename): pickle.dump(stats, f, protocol=pickle.HIGHEST_PROTOCOL) +def _call_with_diagnostics(options, func, *args, **kwargs): + if options.debug: + if isinstance(func, MethodType): + obj = func.__self__ + func_repr = ( + '{0.__module__}.{0.__qualname__}(...).{1.__name__}' + .format(type(obj), func.__func__)) + else: + func_repr = '{0.__module__}.{0.__qualname__}'.format(func) + args_repr = dedent(' ' + pformat(args)[len('['):-len(']')]) + lprefix = len('namespace(') + kwargs_repr = dedent( + ' ' * lprefix + + pformat(SimpleNamespace(**kwargs))[lprefix:-len(')')]) + if args_repr and kwargs_repr: + all_args_repr = f'{args_repr},\n{kwargs_repr}' + else: + all_args_repr = args_repr or kwargs_repr + if all_args_repr: + call = '{}(\n{})'.format( + func_repr, indent(all_args_repr, ' ')) + else: + call = func_repr + '()' + diagnostics.log.debug(f'Calling: {call}') + if options.dryrun: + return + return func(*args, **kwargs) + + def _main(options, module=False): """ Called by :py:func:`main()` for the actual execution and profiling of code; not to be invoked on its own. """ - def call_with_diagnostics(func, *args, **kwargs): - if options.debug: - if isinstance(func, MethodType): - obj = func.__self__ - func_repr = ( - '{0.__module__}.{0.__qualname__}(...).{1.__name__}' - .format(type(obj), func.__func__)) - else: - func_repr = '{0.__module__}.{0.__qualname__}'.format(func) - args_repr = dedent(' ' + pformat(args)[len('['):-len(']')]) - lprefix = len('namespace(') - kwargs_repr = dedent( - ' ' * lprefix - + pformat(SimpleNamespace(**kwargs))[lprefix:-len(')')]) - if args_repr and kwargs_repr: - all_args_repr = f'{args_repr},\n{kwargs_repr}' - else: - all_args_repr = args_repr or kwargs_repr - if all_args_repr: - call = '{}(\n{})'.format( - func_repr, indent(all_args_repr, ' ')) - else: - call = func_repr + '()' - diagnostics.log.debug(f'Calling: {call}') - if options.dryrun: - return - return func(*args, **kwargs) if not options.outfile: extension = 'lprof' if options.line_by_line else 'prof' @@ -1017,21 +1019,24 @@ def call_with_diagnostics(func, *args, **kwargs): if options.prof_mod and options.line_by_line: from line_profiler.autoprofile import autoprofile - call_with_diagnostics( + _call_with_diagnostics( + options, autoprofile.run, script_file, ns, prof_mod=options.prof_mod, profile_imports=options.prof_imports, as_module=module is not None) elif module and options.builtin: - call_with_diagnostics(rmod, options.script, ns) + _call_with_diagnostics(options, rmod, options.script, ns) elif options.builtin: - call_with_diagnostics(execfile, script_file, ns, ns) + _call_with_diagnostics(options, execfile, script_file, ns, ns) elif module: - call_with_diagnostics( + _call_with_diagnostics( + options, prof.runctx, f'rmod({options.script!r}, globals())', ns, ns) else: - call_with_diagnostics( + _call_with_diagnostics( + options, prof.runctx, f'execfile({script_file!r}, globals())', ns, ns) except (KeyboardInterrupt, SystemExit): From 1fdd0da3a2532e5607639ed326278ad5295cc8e3 Mon Sep 17 00:00:00 2001 From: joncrall Date: Sat, 5 Jul 2025 16:39:22 -0400 Subject: [PATCH 69/70] Refactor main to reduce complexity --- kernprof.py | 248 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 143 insertions(+), 105 deletions(-) diff --git a/kernprof.py b/kernprof.py index 97e38b05..90dd50a2 100755 --- a/kernprof.py +++ b/kernprof.py @@ -486,30 +486,87 @@ def pre_parse_single_arg_directive(args, flag, sep='--'): return args, None, [] if i_flag == len(args) - 1: # Last element raise ValueError(f'argument expected for the {flag} option') - return args[:i_flag], args[i_flag + 1], args[i_flag + 2:] + args, thing, post_args = args[:i_flag], args[i_flag + 1], args[i_flag + 2:] + return args, thing, post_args -@_restore.sequence(sys.argv) -@_restore.sequence(sys.path) -@_restore.instance_dict(diagnostics) -def main(args=None): - """ - Runs the command line interface +def positive_float(value): + val = float(value) + if val <= 0: + raise ArgumentError + return val - Note: - To help with traceback formatting, the deletion of temporary - files created during execution may be deferred to when the - interpreter exits. - """ - def positive_float(value): - val = float(value) - if val <= 0: - raise ArgumentError - return val - def no_op(*_, **__) -> None: - pass +def no_op(*_, **__) -> None: + pass + +def _add_core_parser_arguments(parser): + """ + Add the core kernprof args to a ArgumentParser + """ + parser.add_argument('-V', '--version', action='version', version=__version__) + parser.add_argument('-l', '--line-by-line', action='store_true', + help='Use the line-by-line profiler instead of cProfile. Implies --builtin.') + parser.add_argument('-b', '--builtin', action='store_true', + help="Put 'profile' in the builtins. Use 'profile.enable()'/'.disable()', " + "'@profile' to decorate functions, or 'with profile:' to profile a " + 'section of code.') + parser.add_argument('-o', '--outfile', + help="Save stats to (default: 'scriptname.lprof' with " + "--line-by-line, 'scriptname.prof' without)") + parser.add_argument('-s', '--setup', + help='Code to execute before the code to profile') + parser.add_argument('-v', '--verbose', '--view', + action='count', default=0, + help='Increase verbosity level. ' + 'At level 1, view the profiling results ' + 'in addition to saving them; ' + 'at level 2, show other diagnostic info.') + parser.add_argument('-q', '--quiet', + action='count', default=0, + help='Decrease verbosity level. ' + 'At level -1, disable helpful messages ' + '(e.g. "Wrote profile results to <...>"); ' + 'at level -2, silence the stdout; ' + 'at level -3, silence the stderr.') + parser.add_argument('-r', '--rich', action='store_true', + help='Use rich formatting if viewing output') + parser.add_argument('-u', '--unit', default='1e-6', type=positive_float, + help='Output unit (in seconds) in which the timing info is ' + 'displayed (default: 1e-6)') + parser.add_argument('-z', '--skip-zero', action='store_true', + help="Hide functions which have not been called") + parser.add_argument('-i', '--output-interval', type=int, default=0, const=0, nargs='?', + help="Enables outputting of cumulative profiling results to file every n seconds. Uses the threading module. " + "Minimum value is 1 (second). Defaults to disabled.") + parser.add_argument('-p', '--prof-mod', + action='append', + metavar=("{path/to/script | object.dotted.path}" + "[,...]"), + help="List of modules, functions and/or classes " + "to profile specified by their name or path. " + "These profiling targets can be supplied both as " + "comma-separated items, or separately with " + "multiple copies of this flag. " + "Packages are automatically recursed into unless " + "they are specified with `.__init__`. " + "Adding the current script/module profiles the " + "entirety of it. " + "Only works with line_profiler -l, --line-by-line.") + parser.add_argument('--no-preimports', + action='store_true', + help="Instead of eagerly importing all profiling " + "targets specified via -p and profiling them, " + "only profile those that are directly imported in " + "the profiled code. " + "Only works with line_profiler -l, --line-by-line.") + parser.add_argument('--prof-imports', action='store_true', + help="If specified, modules specified to `--prof-mod` will also autoprofile modules that they import. " + "Only works with line_profiler -l, --line-by-line") + + +def _build_parsers(args=None): parser_kwargs = { 'description': 'Run and profile a python script.', } @@ -546,65 +603,7 @@ def no_op(*_, **__) -> None: help_parser = ArgumentParser(**parser_kwargs) parsers = [real_parser, help_parser] for parser in parsers: - parser.add_argument('-V', '--version', action='version', version=__version__) - parser.add_argument('-l', '--line-by-line', action='store_true', - help='Use the line-by-line profiler instead of cProfile. Implies --builtin.') - parser.add_argument('-b', '--builtin', action='store_true', - help="Put 'profile' in the builtins. Use 'profile.enable()'/'.disable()', " - "'@profile' to decorate functions, or 'with profile:' to profile a " - 'section of code.') - parser.add_argument('-o', '--outfile', - help="Save stats to (default: 'scriptname.lprof' with " - "--line-by-line, 'scriptname.prof' without)") - parser.add_argument('-s', '--setup', - help='Code to execute before the code to profile') - parser.add_argument('-v', '--verbose', '--view', - action='count', default=0, - help='Increase verbosity level. ' - 'At level 1, view the profiling results ' - 'in addition to saving them; ' - 'at level 2, show other diagnostic info.') - parser.add_argument('-q', '--quiet', - action='count', default=0, - help='Decrease verbosity level. ' - 'At level -1, disable helpful messages ' - '(e.g. "Wrote profile results to <...>"); ' - 'at level -2, silence the stdout; ' - 'at level -3, silence the stderr.') - parser.add_argument('-r', '--rich', action='store_true', - help='Use rich formatting if viewing output') - parser.add_argument('-u', '--unit', default='1e-6', type=positive_float, - help='Output unit (in seconds) in which the timing info is ' - 'displayed (default: 1e-6)') - parser.add_argument('-z', '--skip-zero', action='store_true', - help="Hide functions which have not been called") - parser.add_argument('-i', '--output-interval', type=int, default=0, const=0, nargs='?', - help="Enables outputting of cumulative profiling results to file every n seconds. Uses the threading module. " - "Minimum value is 1 (second). Defaults to disabled.") - parser.add_argument('-p', '--prof-mod', - action='append', - metavar=("{path/to/script | object.dotted.path}" - "[,...]"), - help="List of modules, functions and/or classes " - "to profile specified by their name or path. " - "These profiling targets can be supplied both as " - "comma-separated items, or separately with " - "multiple copies of this flag. " - "Packages are automatically recursed into unless " - "they are specified with `.__init__`. " - "Adding the current script/module profiles the " - "entirety of it. " - "Only works with line_profiler -l, --line-by-line.") - parser.add_argument('--no-preimports', - action='store_true', - help="Instead of eagerly importing all profiling " - "targets specified via -p and profiling them, " - "only profile those that are directly imported in " - "the profiled code. " - "Only works with line_profiler -l, --line-by-line.") - parser.add_argument('--prof-imports', action='store_true', - help="If specified, modules specified to `--prof-mod` will also autoprofile modules that they import. " - "Only works with line_profiler -l, --line-by-line") + _add_core_parser_arguments(parser) if parser is help_parser or module is literal_code is None: parser.add_argument('script', @@ -613,6 +612,20 @@ def no_op(*_, **__) -> None: help='The python script file, module, or ' 'literal code to run') parser.add_argument('args', nargs='...', help='Optional script arguments') + special_info = { + 'module': module, + 'literal_code': literal_code, + 'post_args': post_args, + 'args': args, + } + return real_parser, help_parser, special_info + + +def _parse_arguments(real_parser, help_parser, special_info, args): + + module = special_info['module'] + literal_code = special_info['literal_code'] + post_args = special_info['post_args'] # Hand off to the dummy parser if necessary to generate the help # text @@ -665,6 +678,28 @@ def no_op(*_, **__) -> None: options.rich = False diagnostics.log.debug('`rich` not installed, unsetting --rich') + return options, tempfile_source_and_content + + +@_restore.sequence(sys.argv) +@_restore.sequence(sys.path) +@_restore.instance_dict(diagnostics) +def main(args=None): + """ + Runs the command line interface + + Note: + To help with traceback formatting, the deletion of temporary + files created during execution may be deferred to when the + interpreter exits. + """ + real_parser, help_parser, special_info = _build_parsers(args=args) + args = special_info['args'] + module = special_info['module'] + + options, tempfile_source_and_content = _parse_arguments( + real_parser, help_parser, special_info, args) + if module is not None: diagnostics.log.debug(f'Profiling module: {module}') elif tempfile_source_and_content: @@ -698,7 +733,7 @@ def no_op(*_, **__) -> None: cleanup() raise try: - _main(options, module) + _profile_module(options, module) except BaseException: # Defer deletion to after the traceback has been formatted # if needs be @@ -890,40 +925,44 @@ def _dump_filtered_stats(tmpdir, prof, filename): pickle.dump(stats, f, protocol=pickle.HIGHEST_PROTOCOL) +def _format_call_message(func, *args, **kwargs): + if isinstance(func, MethodType): + obj = func.__self__ + func_repr = ( + '{0.__module__}.{0.__qualname__}(...).{1.__name__}' + .format(type(obj), func.__func__)) + else: + func_repr = '{0.__module__}.{0.__qualname__}'.format(func) + args_repr = dedent(' ' + pformat(args)[len('['):-len(']')]) + lprefix = len('namespace(') + kwargs_repr = dedent( + ' ' * lprefix + + pformat(SimpleNamespace(**kwargs))[lprefix:-len(')')]) + if args_repr and kwargs_repr: + all_args_repr = f'{args_repr},\n{kwargs_repr}' + else: + all_args_repr = args_repr or kwargs_repr + if all_args_repr: + call = '{}(\n{})'.format( + func_repr, indent(all_args_repr, ' ')) + else: + call = func_repr + '()' + return call + + def _call_with_diagnostics(options, func, *args, **kwargs): if options.debug: - if isinstance(func, MethodType): - obj = func.__self__ - func_repr = ( - '{0.__module__}.{0.__qualname__}(...).{1.__name__}' - .format(type(obj), func.__func__)) - else: - func_repr = '{0.__module__}.{0.__qualname__}'.format(func) - args_repr = dedent(' ' + pformat(args)[len('['):-len(']')]) - lprefix = len('namespace(') - kwargs_repr = dedent( - ' ' * lprefix - + pformat(SimpleNamespace(**kwargs))[lprefix:-len(')')]) - if args_repr and kwargs_repr: - all_args_repr = f'{args_repr},\n{kwargs_repr}' - else: - all_args_repr = args_repr or kwargs_repr - if all_args_repr: - call = '{}(\n{})'.format( - func_repr, indent(all_args_repr, ' ')) - else: - call = func_repr + '()' + call = _format_call_message(func, *args, **kwargs) diagnostics.log.debug(f'Calling: {call}') if options.dryrun: return return func(*args, **kwargs) -def _main(options, module=False): +def _profile_module(options, module=False): """ - Called by :py:func:`main()` for the actual execution and profiling - of code; - not to be invoked on its own. + Called by :py:func:`main()` for the actual execution and profiling of code + after resolving options; not to be invoked on its own. """ if not options.outfile: @@ -1018,7 +1057,6 @@ def _main(options, module=False): 'prof': prof} if options.prof_mod and options.line_by_line: from line_profiler.autoprofile import autoprofile - _call_with_diagnostics( options, autoprofile.run, script_file, ns, From 4a420dbd9c19a53cc6eda5b09bbea81d5f51232d Mon Sep 17 00:00:00 2001 From: joncrall Date: Sat, 5 Jul 2025 16:53:33 -0400 Subject: [PATCH 70/70] Breakup _profile_main into smaller parts to reduce complexity --- kernprof.py | 102 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 60 insertions(+), 42 deletions(-) diff --git a/kernprof.py b/kernprof.py index 90dd50a2..a11cb2ad 100755 --- a/kernprof.py +++ b/kernprof.py @@ -733,7 +733,7 @@ def main(args=None): cleanup() raise try: - _profile_module(options, module) + _main_profile(options, module) except BaseException: # Defer deletion to after the traceback has been formatted # if needs be @@ -959,12 +959,13 @@ def _call_with_diagnostics(options, func, *args, **kwargs): return func(*args, **kwargs) -def _profile_module(options, module=False): - """ - Called by :py:func:`main()` for the actual execution and profiling of code - after resolving options; not to be invoked on its own. +def _pre_profile(options, module): """ + Prepare the environment to execute profiling with requested options. + Note: + modifies ``options`` with extra attributes. + """ if not options.outfile: extension = 'lprof' if options.line_by_line else 'prof' options.outfile = f'{os.path.basename(options.script)}.{extension}' @@ -1042,12 +1043,22 @@ def _profile_module(options, module=False): exclude = set() if module else {script_file} _write_preimports(prof, options, exclude) - use_timer = options.output_interval and not options.dryrun - if use_timer: - rt = RepeatedTimer(max(options.output_interval, 1), prof.dump_stats, options.outfile) - original_stdout = sys.stdout - if use_timer: - rt = RepeatedTimer(max(options.output_interval, 1), prof.dump_stats, options.outfile) + options.global_profiler = global_profiler + options.install_profiler = install_profiler + if options.output_interval and not options.dryrun: + options.rt = RepeatedTimer(max(options.output_interval, 1), prof.dump_stats, options.outfile) + else: + options.rt = None + options.original_stdout = sys.stdout + return script_file, prof + + +def _main_profile(options, module=False): + """ + Called by :py:func:`main()` for the actual execution and profiling of code + after initial parsing of options; not to be invoked on its own. + """ + script_file, prof = _pre_profile(options, module) try: try: rmod = functools.partial(run_module, @@ -1080,38 +1091,45 @@ def _profile_module(options, module=False): except (KeyboardInterrupt, SystemExit): pass finally: - if use_timer: - rt.stop() - if not options.dryrun: - _dump_filtered_stats(options.tmpdir, prof, options.outfile) - diagnostics.log.info( - ('Profile results would have been written to ' - if options.dryrun else - 'Wrote profile results ') - + f'to {options.outfile!r}') - if options.verbose > 0 and not options.dryrun: - if isinstance(prof, ContextualProfile): - prof.print_stats() - else: - prof.print_stats(output_unit=options.unit, - stripzeros=options.skip_zero, - rich=options.rich, - stream=original_stdout) + _post_profile(options, prof) + + +def _post_profile(options, prof): + """ + Cleanup setup after executing a main profile + """ + if options.rt is not None: + options.rt.stop() + if not options.dryrun: + _dump_filtered_stats(options.tmpdir, prof, options.outfile) + diagnostics.log.info( + ('Profile results would have been written to ' + if options.dryrun else + 'Wrote profile results ') + + f'to {options.outfile!r}') + if options.verbose > 0 and not options.dryrun: + if isinstance(prof, ContextualProfile): + prof.print_stats() else: - py_exe = _python_command() - if isinstance(prof, ContextualProfile): - show_mod = 'pstats' - else: - show_mod = 'line_profiler -rmt' - diagnostics.log.info('Inspect results with:\n' - f'{quote(py_exe)} -m {show_mod} ' - f'{quote(options.outfile)}') - # Fully disable the profiler - for _ in range(prof.enable_count): - prof.disable_by_count() - # Restore the state of the global `@line_profiler.profile` - if global_profiler: - install_profiler(None) + prof.print_stats(output_unit=options.unit, + stripzeros=options.skip_zero, + rich=options.rich, + stream=options.original_stdout) + else: + py_exe = _python_command() + if isinstance(prof, ContextualProfile): + show_mod = 'pstats' + else: + show_mod = 'line_profiler -rmt' + diagnostics.log.info('Inspect results with:\n' + f'{quote(py_exe)} -m {show_mod} ' + f'{quote(options.outfile)}') + # Fully disable the profiler + for _ in range(prof.enable_count): + prof.disable_by_count() + # Restore the state of the global `@line_profiler.profile` + if options.global_profiler: + options.install_profiler(None) if __name__ == '__main__':