From 845ba1ad6eb77cdedb73ee38cfbb92c2a0f4bbff Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 29 Apr 2025 18:49:05 +0200 Subject: [PATCH 1/4] Fixed `kernprof -m` CHANGELOG.rst Added entry kernprof.py::main() Now calling `runpy.run_module()` with `alter_sys = True` to maintain compatibility with code expecting the executed module to be found at `sys.modules['__main__']` line_profiler/autoprofile/autoprofile.py::run() Updated implementation to do the equivalent of `runpy.run_module(alter_sys=True)` tests/test_kernprof.py test_kernprof_m_parsing() - Updated implementation to expect the correct `sys.argv[0]` (real path to the module file) - Replaced `tempfile.mkdtemp()` with `tempfile.TemporaryDirectory` test_kernprof_m_sys_modules() New test for compatibility with tools (e.g. `@enum.global_enum`) expecting the executed module to be found at `sys.modules['__main__']` --- CHANGELOG.rst | 1 + kernprof.py | 9 ++- line_profiler/autoprofile/autoprofile.py | 44 +++++++++++- tests/test_kernprof.py | 86 +++++++++++++++++++----- 4 files changed, 114 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0e242a51..c30a0f65 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,6 +9,7 @@ Changes * FIX: Fixed auto-profiling of async function definitions #330 * ENH: Added CLI argument ``-m`` to ``kernprof`` for running a library module as a script; also made it possible for profiling targets to be supplied across multiple ``-p`` flags * FIX: Fixed explicit profiling of class methods; added handling for profiling static, bound, and partial methods, ``functools.partial`` objects, (cached) properties, and async generator functions +* FIX: Fixed namespace bug when running ``kernprof -m`` on certain modules (e.g. ``calendar`` on Python 3.12+) 4.2.0 ~~~~~ diff --git a/kernprof.py b/kernprof.py index ef096486..b8e558cf 100755 --- a/kernprof.py +++ b/kernprof.py @@ -470,7 +470,8 @@ def positive_float(value): try: try: execfile_ = execfile - rmod_ = run_module + rmod_ = functools.partial(run_module, + run_name='__main__', alter_sys=True) ns = locals() if options.prof_mod and options.line_by_line: from line_profiler.autoprofile import autoprofile @@ -487,13 +488,11 @@ def positive_float(value): profile_imports=options.prof_imports, as_module=module is not None) elif module and options.builtin: - run_module(options.script, ns, '__main__') + rmod_(options.script, ns) elif options.builtin: execfile(script_file, ns, ns) elif module: - prof.runctx(f'rmod_({options.script!r}, globals(), "__main__")', - ns, - ns) + prof.runctx(f'rmod_({options.script!r}, globals())', ns, ns) else: prof.runctx('execfile_(%r, globals())' % (script_file,), ns, ns) except (KeyboardInterrupt, SystemExit): diff --git a/line_profiler/autoprofile/autoprofile.py b/line_profiler/autoprofile/autoprofile.py index 7b738c86..aef36604 100644 --- a/line_profiler/autoprofile/autoprofile.py +++ b/line_profiler/autoprofile/autoprofile.py @@ -44,11 +44,16 @@ def main(): python -m kernprof -p demo.py -l demo.py python -m line_profiler -rmt demo.py.lprof """ - +import contextlib +import functools +import importlib.util +import operator +import sys import types from .ast_tree_profiler import AstTreeProfiler from .run_module import AstTreeModuleProfiler from .line_profiler_utils import add_imported_function_or_module +from .util_static import modpath_to_modname PROFILER_LOCALS_NAME = 'prof' @@ -92,10 +97,43 @@ def run(script_file, ns, prof_mod, profile_imports=False, as_module=False): as_module (bool): Whether we're running script_file as a module """ - Profiler = AstTreeModuleProfiler if as_module else AstTreeProfiler + @contextlib.contextmanager + def restore_dict(d, target=None): + copy = d.copy() + yield target + d.clear() + d.update(copy) + + if as_module: + Profiler = AstTreeModuleProfiler + module_name = modpath_to_modname(script_file) + assert module_name is not None + + module_obj = types.ModuleType(module_name) + namespace = vars(module_obj) + namespace.update(ns) + + # Set the `__spec__` correctly + spec = getattr(sys.modules.get(module_name), '__spec__', None) + if spec is None: + spec = importlib.util.find_spec(module_name) + module_obj.__spec__ = spec + + # Set the module object to `sys.modules` via a callback, and + # then restore it via the context manager + callback = functools.partial( + operator.setitem, sys.modules, '__main__', module_obj) + ctx = restore_dict(sys.modules, callback) + else: + Profiler = AstTreeProfiler + namespace = ns + ctx = contextlib.nullcontext(lambda: None) + profiler = Profiler(script_file, prof_mod, profile_imports) tree_profiled = profiler.profile() _extend_line_profiler_for_profiling_imports(ns[PROFILER_LOCALS_NAME]) code_obj = compile(tree_profiled, script_file, 'exec') - exec(code_obj, ns, ns) + with ctx as callback: + callback() + exec(code_obj, namespace, namespace) diff --git a/tests/test_kernprof.py b/tests/test_kernprof.py index 98a1e5cb..2f94f6da 100644 --- a/tests/test_kernprof.py +++ b/tests/test_kernprof.py @@ -1,3 +1,4 @@ +import shlex import sys import tempfile import unittest @@ -24,14 +25,14 @@ def g(x): 'use_kernprof_exec, args, expected_output, expect_error', [([False, ['-m'], '', True]), # `python -m kernprof` - (False, ['-m', 'mymod'], "['mymod']", False), + (False, ['-m', 'mymod'], "[__MYMOD__]", False), # `kernprof` - (True, ['-m', 'mymod'], "['mymod']", False), - (False, ['-m', 'mymod', '-p', 'bar'], "['mymod', '-p', 'bar']", False), + (True, ['-m', 'mymod'], "[__MYMOD__]", False), + (False, ['-m', 'mymod', '-p', 'bar'], "[__MYMOD__, '-p', 'bar']", False), # `-p bar` consumed by `kernprof`, `-p baz` are not (False, ['-p', 'bar', '-m', 'mymod', '-p', 'baz'], - "['mymod', '-p', 'baz']", + "[__MYMOD__, '-p', 'baz']", False), # Separator `--` broke off the remainder, so the requisite arg for # `-m` is not found and we error out @@ -49,28 +50,76 @@ def test_kernprof_m_parsing( an argument and cuts off everything after it, passing that along to the module to be executed. """ - temp_dpath = ub.Path(tempfile.mkdtemp()) - (temp_dpath / 'mymod.py').write_text(ub.codeblock( - ''' - import sys - - - if __name__ == '__main__': - print(sys.argv) - ''')) - if use_kernprof_exec: - cmd = ['kernprof'] - else: - cmd = [sys.executable, '-m', 'kernprof'] - proc = ub.cmd(cmd + args, cwd=temp_dpath, verbose=2) + with tempfile.TemporaryDirectory() as tmpdir: + temp_dpath = ub.Path(tmpdir) + mod = (temp_dpath / 'mymod.py').resolve() + mod.write_text(ub.codeblock( + ''' + import sys + + + if __name__ == '__main__': + print(sys.argv) + ''')) + if use_kernprof_exec: + cmd = ['kernprof'] + else: + cmd = [sys.executable, '-m', 'kernprof'] + proc = ub.cmd(cmd + args, cwd=temp_dpath, verbose=2) if expect_error: assert proc.returncode return else: proc.check_returncode() + expected_output = expected_output.replace('__MYMOD__', repr(str(mod))) assert proc.stdout.startswith(expected_output) +@pytest.mark.skipif(sys.version_info[:2] < (3, 11), + reason='no `@enum.global_enum` in Python ' + f'{".".join(str(v) for v in sys.version_info[:3])}') +@pytest.mark.parametrize(('flags', 'profiled_main'), + [('-lv -p mymod', True), # w/autoprofile + ('-lv', True), # w/o autoprofile + ('-b', False)]) # w/o line profiling +def test_kernprof_m_sys_modules(flags, profiled_main): + """ + Test that `kernprof -m` is amenable to modules relying on the global + `sys` state (e.g. those using `@enum.global_enum`). + """ + with tempfile.TemporaryDirectory() as tmpdir: + temp_dpath = ub.Path(tmpdir) + (temp_dpath / 'mymod.py').write_text(ub.codeblock( + ''' + import enum + import os + import sys + + + @enum.global_enum + class MyEnum(enum.Enum): + FOO = 1 + BAR = 2 + + + @profile + def main(): + x = FOO.value + BAR.value + print(x) + assert x == 3 + + + if __name__ == '__main__': + main() + ''')) + cmd = [sys.executable, '-m', 'kernprof', + *shlex.split(flags), '-m', 'mymod'] + proc = ub.cmd(cmd, cwd=temp_dpath, verbose=2) + proc.check_returncode() + assert proc.stdout.startswith('3\n') + assert ('Function: main' in proc.stdout) == profiled_main + + class TestKernprof(unittest.TestCase): def test_enable_disable(self): @@ -130,6 +179,7 @@ def test_gen_decorator(self): next(i) self.assertEqual(profile.enable_count, 0) + if __name__ == '__main__': """ CommandLine: From 51b4a81fa230ab2cf32116102b62e3fcd5610837 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 29 Apr 2025 19:47:05 +0200 Subject: [PATCH 2/4] CI-friendly fix --- tests/test_kernprof.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_kernprof.py b/tests/test_kernprof.py index 2f94f6da..cfb7a553 100644 --- a/tests/test_kernprof.py +++ b/tests/test_kernprof.py @@ -1,3 +1,5 @@ +import os +import re import shlex import sys import tempfile @@ -71,8 +73,9 @@ def test_kernprof_m_parsing( return else: proc.check_returncode() - expected_output = expected_output.replace('__MYMOD__', repr(str(mod))) - assert proc.stdout.startswith(expected_output) + expected_output = re.escape(expected_output).replace( + '__MYMOD__', "'.*{}'".format(re.escape(os.path.sep + mod.name))) + assert re.match('^' + expected_output, proc.stdout) @pytest.mark.skipif(sys.version_info[:2] < (3, 11), From a51d4464b356f33c5b1a4b76be66e7b656f413ce Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 3 May 2025 20:23:41 +0200 Subject: [PATCH 3/4] Stylistic updates line_profiler/autoprofile/autoprofile.py::run() - Replaced assertion with an explicitly raised `ModuleNotFoundError` - Streamlined logic to always call `importlib.util.find_spec()`, regardless of whether the module is found in `sys.modules` and already has a spec --- line_profiler/autoprofile/autoprofile.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/line_profiler/autoprofile/autoprofile.py b/line_profiler/autoprofile/autoprofile.py index aef36604..d5894318 100644 --- a/line_profiler/autoprofile/autoprofile.py +++ b/line_profiler/autoprofile/autoprofile.py @@ -107,17 +107,16 @@ def restore_dict(d, target=None): if as_module: Profiler = AstTreeModuleProfiler module_name = modpath_to_modname(script_file) - assert module_name is not None + if not module_name: + raise ModuleNotFoundError(f'script_file = {script_file!r}: ' + 'cannot find corresponding module') module_obj = types.ModuleType(module_name) namespace = vars(module_obj) namespace.update(ns) # Set the `__spec__` correctly - spec = getattr(sys.modules.get(module_name), '__spec__', None) - if spec is None: - spec = importlib.util.find_spec(module_name) - module_obj.__spec__ = spec + module_obj.__spec__ = importlib.util.find_spec(module_name) # Set the module object to `sys.modules` via a callback, and # then restore it via the context manager From 53a384692da2283400f0f6576d2226867a43e87e Mon Sep 17 00:00:00 2001 From: joncrall Date: Sun, 4 May 2025 18:17:40 -0400 Subject: [PATCH 4/4] try to force CI to run --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c30a0f65..62c53494 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,7 +9,7 @@ Changes * FIX: Fixed auto-profiling of async function definitions #330 * ENH: Added CLI argument ``-m`` to ``kernprof`` for running a library module as a script; also made it possible for profiling targets to be supplied across multiple ``-p`` flags * FIX: Fixed explicit profiling of class methods; added handling for profiling static, bound, and partial methods, ``functools.partial`` objects, (cached) properties, and async generator functions -* FIX: Fixed namespace bug when running ``kernprof -m`` on certain modules (e.g. ``calendar`` on Python 3.12+) +* FIX: Fixed namespace bug when running ``kernprof -m`` on certain modules (e.g. ``calendar`` on Python 3.12+). 4.2.0 ~~~~~