From 5dc278cf577dc010f1eb41403881db5e8d865015 Mon Sep 17 00:00:00 2001 From: Matthias Diener Date: Mon, 31 Mar 2025 17:23:49 -0500 Subject: [PATCH 1/5] PythonFunctionGenerator: make compatible with linecache use linecache directly only adjust filename if line_profiler loaded improve args fix maintain compatibility with pudb broaden line_profiler usage check more type annotations pyright ignores rename generated functions to avoid linecache warnings remove special pudb handling make sure names are unique fix bpr expand to support PythonCodeGenerator, refactor fixes remove special line_profiler handling Relevant PR (https://github.com/pyutils/line_profiler/pull/341) has been merged Rework for linecache compatibility, improve tests Co-authored-by: Andreas Kloeckner --- .basedpyright/baseline.json | 804 +------------------------------- pytools/__init__.py | 16 +- pytools/py_codegen.py | 203 ++++++-- pytools/test/test_py_codegen.py | 52 +++ 4 files changed, 222 insertions(+), 853 deletions(-) diff --git a/.basedpyright/baseline.json b/.basedpyright/baseline.json index f8f6a765..528606ce 100644 --- a/.basedpyright/baseline.json +++ b/.basedpyright/baseline.json @@ -14228,778 +14228,10 @@ ], "./pytools/py_codegen.py": [ { - "code": "reportUnusedImport", - "range": { - "startColumn": 4, - "endColumn": 15, - "lineCount": 1 - } - }, - { - "code": "reportUnusedImport", - "range": { - "startColumn": 4, - "endColumn": 29, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 8, - "endColumn": 18, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 25, - "endColumn": 29, - "lineCount": 1 - } - }, - { - "code": "reportMissingParameterType", - "range": { - "startColumn": 25, - "endColumn": 29, - "lineCount": 1 - } - }, - { - "code": "reportUnknownArgumentType", - "range": { - "startColumn": 39, - "endColumn": 43, - "lineCount": 1 - } - }, - { - "code": "reportUnknownArgumentType", - "range": { - "startColumn": 16, - "endColumn": 27, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 15, - "endColumn": 26, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 35, - "endColumn": 39, - "lineCount": 1 - } - }, - { - "code": "reportMissingParameterType", - "range": { - "startColumn": 35, - "endColumn": 39, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 31, - "endColumn": 46, - "lineCount": 1 - } - }, - { - "code": "reportUnknownArgumentType", - "range": { - "startColumn": 31, - "endColumn": 57, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 23, - "endColumn": 27, - "lineCount": 1 - } - }, - { - "code": "reportMissingParameterType", - "range": { - "startColumn": 23, - "endColumn": 27, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 29, - "endColumn": 33, - "lineCount": 1 - } - }, - { - "code": "reportMissingParameterType", - "range": { - "startColumn": 29, - "endColumn": 33, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 35, - "endColumn": 45, - "lineCount": 1 - } - }, - { - "code": "reportMissingParameterType", - "range": { - "startColumn": 35, - "endColumn": 45, - "lineCount": 1 - } - }, - { - "code": "reportUnannotatedClassAttribute", - "range": { - "startColumn": 13, - "endColumn": 17, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 16, - "endColumn": 25, - "lineCount": 1 - } - }, - { - "code": "reportUnknownArgumentType", - "range": { - "startColumn": 21, - "endColumn": 30, - "lineCount": 1 - } - }, - { - "code": "reportUnknownArgumentType", - "range": { - "startColumn": 34, - "endColumn": 38, - "lineCount": 1 - } - }, - { - "code": "reportUnknownArgumentType", - "range": { - "startColumn": 50, - "endColumn": 54, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 8, - "endColumn": 20, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 15, - "endColumn": 30, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 15, - "endColumn": 66, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 17, - "endColumn": 42, - "lineCount": 1 - } - }, - { - "code": "reportUnknownArgumentType", - "range": { - "startColumn": 41, - "endColumn": 50, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 4, - "endColumn": 26, - "lineCount": 1 - } - }, - { - "code": "reportUnknownArgumentType", - "range": { - "startColumn": 54, - "endColumn": 65, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 11, - "endColumn": 22, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 0, - "endColumn": 18, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 23, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportMissingParameterType", - "range": { - "startColumn": 23, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportUnannotatedClassAttribute", - "range": { - "startColumn": 13, - "endColumn": 24, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 8, - "endColumn": 20, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 12, - "endColumn": 13, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 15, - "endColumn": 16, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 20, - "endColumn": 36, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 20, - "endColumn": 42, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 15, - "endColumn": 76, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 27, - "endColumn": 30, - "lineCount": 1 - } - }, - { - "code": "reportMissingParameterType", - "range": { - "startColumn": 27, - "endColumn": 30, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 12, - "endColumn": 17, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 19, - "endColumn": 28, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 30, - "endColumn": 48, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 12, - "endColumn": 17, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 19, - "endColumn": 28, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 30, - "endColumn": 37, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 39, - "endColumn": 57, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 8, - "endColumn": 19, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 8, - "endColumn": 26, - "lineCount": 1 - } - }, - { - "code": "reportUnknownArgumentType", - "range": { - "startColumn": 27, - "endColumn": 45, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 12, - "endColumn": 13, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 15, - "endColumn": 23, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 27, - "endColumn": 40, - "lineCount": 1 - } - }, - { - "code": "reportUnknownArgumentType", - "range": { - "startColumn": 43, - "endColumn": 51, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 12, - "endColumn": 13, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 16, - "endColumn": 20, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 22, - "endColumn": 32, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 34, - "endColumn": 41, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 46, - "endColumn": 61, - "lineCount": 1 - } - }, - { - "code": "reportAny", - "range": { - "startColumn": 20, - "endColumn": 45, - "lineCount": 1 - } - }, - { - "code": "reportUnknownArgumentType", - "range": { - "startColumn": 34, - "endColumn": 44, - "lineCount": 1 - } - }, - { - "code": "reportUnknownArgumentType", - "range": { - "startColumn": 47, - "endColumn": 58, - "lineCount": 1 - } - }, - { - "code": "reportUnknownArgumentType", - "range": { - "startColumn": 65, - "endColumn": 69, - "lineCount": 1 - } - }, - { - "code": "reportUnknownArgumentType", - "range": { - "startColumn": 28, - "endColumn": 35, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 8, - "endColumn": 24, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 23, - "endColumn": 29, - "lineCount": 1 - } - }, - { - "code": "reportMissingParameterType", - "range": { - "startColumn": 23, - "endColumn": 29, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 31, - "endColumn": 35, - "lineCount": 1 - } - }, - { - "code": "reportMissingParameterType", - "range": { - "startColumn": 31, - "endColumn": 35, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 8, - "endColumn": 24, - "lineCount": 1 - } - }, - { - "code": "reportUnknownArgumentType", - "range": { - "startColumn": 25, - "endColumn": 31, - "lineCount": 1 - } - }, - { - "code": "reportUnknownArgumentType", - "range": { - "startColumn": 33, - "endColumn": 37, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 26, - "endColumn": 32, - "lineCount": 1 - } - }, - { - "code": "reportMissingParameterType", - "range": { - "startColumn": 26, - "endColumn": 32, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 34, - "endColumn": 38, - "lineCount": 1 - } - }, - { - "code": "reportMissingParameterType", - "range": { - "startColumn": 34, - "endColumn": 38, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 8, - "endColumn": 19, - "lineCount": 1 - } - }, - { - "code": "reportUnannotatedClassAttribute", - "range": { - "startColumn": 13, - "endColumn": 19, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 8, - "endColumn": 17, - "lineCount": 1 - } - }, - { - "code": "reportUnannotatedClassAttribute", - "range": { - "startColumn": 13, - "endColumn": 17, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 8, - "endColumn": 17, - "lineCount": 1 - } - }, - { - "code": "reportUnannotatedClassAttribute", - "range": { - "startColumn": 13, - "endColumn": 17, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", + "code": "reportAny", "range": { "startColumn": 20, - "endColumn": 38, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 8, - "endColumn": 16, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 24, - "endColumn": 28, - "lineCount": 1 - } - }, - { - "code": "reportMissingParameterType", - "range": { - "startColumn": 24, - "endColumn": 28, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 32, - "endColumn": 38, - "lineCount": 1 - } - }, - { - "code": "reportMissingParameterType", - "range": { - "startColumn": 32, - "endColumn": 38, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 15, - "endColumn": 24, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 15, - "endColumn": 41, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 8, - "endColumn": 20, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 15, - "endColumn": 57, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 26, - "endColumn": 37, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 47, - "endColumn": 56, + "endColumn": 45, "lineCount": 1 } }, @@ -15019,14 +14251,6 @@ "lineCount": 1 } }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 8, - "endColumn": 24, - "lineCount": 1 - } - }, { "code": "reportUnknownArgumentType", "range": { @@ -17875,14 +17099,6 @@ "lineCount": 1 } }, - { - "code": "reportPrivateUsage", - "range": { - "startColumn": 40, - "endColumn": 53, - "lineCount": 1 - } - }, { "code": "reportUnknownParameterType", "range": { @@ -18341,14 +17557,6 @@ "lineCount": 1 } }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 36, - "endColumn": 59, - "lineCount": 1 - } - }, { "code": "reportAny", "range": { @@ -18389,14 +17597,6 @@ "lineCount": 1 } }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 4, - "endColumn": 5, - "lineCount": 1 - } - }, { "code": "reportUnknownVariableType", "range": { diff --git a/pytools/__init__.py b/pytools/__init__.py index 2c9b56fa..6e28fc6d 100644 --- a/pytools/__init__.py +++ b/pytools/__init__.py @@ -2226,13 +2226,15 @@ def generate_unique_names(prefix): def generate_numbered_unique_names( - prefix: str, num: int | None = None) -> Iterable[tuple[int, str]]: + prefix: str, + num: int | None = None, + suffix: str = "") -> Iterable[tuple[int, str]]: if num is None: - yield (0, prefix) + yield (0, prefix + suffix) num = 0 while True: - name = f"{prefix}_{num}" + name = f"{prefix}_{num}{suffix}" num += 1 yield (num, name) @@ -2254,19 +2256,22 @@ class UniqueNameGenerator: """ def __init__(self, existing_names: Collection[str] | None = None, - forced_prefix: str = ""): + forced_prefix: str = "", + forced_suffix: str = "") -> None: """ Create a new :class:`UniqueNameGenerator`. :arg existing_names: a :class:`set` of existing names that will be skipped when generating new names. :arg forced_prefix: all generated :class:`str` have this prefix. + :arg forced_suffix: all generated :class:`str` have this suffix. """ if existing_names is None: existing_names = set() self.existing_names = set(existing_names) self.forced_prefix = forced_prefix + self.forced_suffix: str = forced_suffix self.prefix_to_counter: dict[str, int] = {} def is_name_conflicting(self, name: str) -> bool: @@ -2326,7 +2331,8 @@ def __call__(self, based_on: str = "id") -> str: # }}} - for counter, var_name in generate_numbered_unique_names(based_on, counter): # noqa: B020,B007 + for counter, var_name in generate_numbered_unique_names( # noqa: B020,B007 + based_on, counter, self.forced_suffix): if not self.is_name_conflicting(var_name): break diff --git a/pytools/py_codegen.py b/pytools/py_codegen.py index 01943d89..3f799ce3 100644 --- a/pytools/py_codegen.py +++ b/pytools/py_codegen.py @@ -23,79 +23,162 @@ THE SOFTWARE. """ + import marshal +from dataclasses import dataclass, field from importlib.util import MAGIC_NUMBER as BYTECODE_VERSION from types import FunctionType, ModuleType +from typing import TYPE_CHECKING, Any, Literal, TypeAlias, cast + + +if TYPE_CHECKING: + from collections.abc import Callable, Iterable -from pytools.codegen import ( # noqa: F401 +from pytools.codegen import ( CodeGenerator as CodeGeneratorBase, Indentation, remove_common_indentation, ) -class PythonCodeGenerator(CodeGeneratorBase): - def get_module(self, name=None): - if name is None: - name = "" +__all__ = ( + "ExistingLineCacheWarning", + "Indentation", + "PicklableFunction", + "PicklableModule", + "PythonCodeGenerator", + "PythonFunctionGenerator", + "remove_common_indentation", +) + + +class ExistingLineCacheWarning(Warning): + """Warning for overwriting existing generated code in the linecache.""" + + +def _linecache_unique_name(name_prefix: str, source_text: str | None) -> str: + import linecache + + if source_text is not None: + from siphash24 import siphash13 # pyright: ignore[reportUnknownVariableType] + src_digest = cast("str", siphash13(source_text.encode()).hexdigest()) # pyright: ignore[reportUnknownMemberType] + + name_prefix = f"{name_prefix}-{src_digest}" + + from pytools import UniqueNameGenerator + name_gen = UniqueNameGenerator( + existing_names=linecache.cache.keys(), + forced_prefix=" dict[str, Any]: + if name_prefix is None: + name_prefix = "module" + + if name is None: + name = _linecache_unique_name(name_prefix, source_text) + + result_dict: dict[str, Any] = {} + + # {{{ insert into linecache + + import linecache - result_dict = {} - source_text = self.get() - exec(compile( - source_text.rstrip()+"\n", name, "exec"), - result_dict) - result_dict["_MODULE_SOURCE_CODE"] = source_text - return result_dict + if name in linecache.cache: + from warnings import warn + warn(f"Overwriting existing generated code in linecache: '{name}'.", + ExistingLineCacheWarning, + stacklevel=2) - def get_picklable_module(self, name=None): - return PicklableModule(self.get_module(name=name)) + linecache.cache[name] = (len(source_text), None, + [e+"\n" for e in source_text.split("\n")], name) + + # }}} + + code_obj = compile( + source_text.rstrip()+"\n", name, "exec") + result_dict["__code__"] = code_obj + exec(code_obj, result_dict) + + return result_dict + + +class PythonCodeGenerator(CodeGeneratorBase): + def get_module(self, name: str | None = None, + *, + name_prefix: str | None = None, + ) -> dict[str, Any]: + return _make_module(self.get(), name=name, name_prefix=name_prefix) + + def get_picklable_module(self, + name: str | None = None, + name_prefix: str | None = None + ) -> PicklableModule: + return PicklableModule(self.get_module(name=name, name_prefix=name_prefix)) class PythonFunctionGenerator(PythonCodeGenerator): - def __init__(self, name, args, decorators=None): - PythonCodeGenerator.__init__(self) + name: str + + def __init__(self, name: str, args: Iterable[str], + decorators: Iterable[str] = ()) -> None: + super().__init__() self.name = name - if decorators: - for decorator in decorators: - self(decorator) + for decorator in decorators: + self(decorator) self("def {}({}):".format(name, ", ".join(args))) self.indent() - @property - def _gen_filename(self): - return f"" + def get_function(self) -> Callable[..., Any]: + return self.get_module(name_prefix=self.name)[self.name] # pyright: ignore[reportAny] - def get_function(self): - return self.get_module(name=self._gen_filename)[self.name] - - def get_picklable_function(self): - module = self.get_picklable_module(name=self._gen_filename) - return PicklableFunction(module, self.name) + def get_picklable_function(self) -> PicklableFunction: + return PicklableFunction( + self.get_picklable_module(name_prefix=self.name), self.name) # {{{ pickling of binaries for generated code -def _get_empty_module_dict(): - result_dict = {} - exec(compile("", "", "exec"), result_dict) +def _get_empty_module_dict(filename: str | None = None) -> dict[str, Any]: + if filename is None: + filename = "" + + result_dict: dict[str, Any] = {} + code_obj = compile("", filename, "exec") + result_dict["__code__"] = code_obj + exec(code_obj, result_dict) return result_dict _empty_module_dict = _get_empty_module_dict() +_FunctionsType: TypeAlias = dict[str, tuple[str, bytes, tuple[object, ...] | None]] +_ModulesType: TypeAlias = dict[str, str] + + +@dataclass class PicklableModule: - def __init__(self, mod_globals): - self.mod_globals = mod_globals + mod_globals: dict[str, Any] + name_prefix: str | None = field(kw_only=True, default=None) + source_code: str | None = field(kw_only=True, default=None) def __getstate__(self): - nondefault_globals = {} - functions = {} - modules = {} + functions: _FunctionsType = {} + modules: _ModulesType = {} + nondefault_globals: dict[str, object] = {} - for k, v in self.mod_globals.items(): + for k, v in self.mod_globals.items(): # pyright: ignore[reportAny] if isinstance(v, FunctionType): functions[k] = ( v.__name__, @@ -106,14 +189,29 @@ def __getstate__(self): elif k not in _empty_module_dict: nondefault_globals[k] = v - return (1, BYTECODE_VERSION, functions, modules, nondefault_globals) - - def __setstate__(self, obj): + return (2, BYTECODE_VERSION, functions, modules, nondefault_globals, + self.name_prefix, self.source_code) + + def __setstate__(self, obj: ( + tuple[Literal[0], bytes, _FunctionsType, dict[str, object]] + | tuple[Literal[1], bytes, _FunctionsType, _ModulesType, + dict[str, object]] + | tuple[Literal[2], bytes, _FunctionsType, _ModulesType, + dict[str, object], str | None, str | None] + ) + ): if obj[0] == 0: magic, functions, nondefault_globals = obj[1:] modules = {} + name_prefix: str | None = None + source_code: str | None = None elif obj[0] == 1: magic, functions, modules, nondefault_globals = obj[1:] + name_prefix = None + source_code = None + elif obj[0] == 2: + magic, functions, modules, nondefault_globals, name_prefix, source_code = \ + obj[1:] else: raise ValueError("unknown version of PicklableModule") @@ -122,9 +220,17 @@ def __setstate__(self, obj): "cannot unpickle function binary: incorrect magic value " f"(got: {magic!r}, expected: {BYTECODE_VERSION!r})") - mod_globals = _empty_module_dict.copy() + unique_filename = _linecache_unique_name( + name_prefix if name_prefix else "module", source_code) + mod_globals = _get_empty_module_dict(unique_filename) mod_globals.update(nondefault_globals) + import linecache + if source_code: + linecache.cache[unique_filename] = (len(source_code), None, + [e+"\n" for e in source_code.split("\n")], + unique_filename) + from importlib import import_module for k, mod_name in modules.items(): mod_globals[k] = import_module(mod_name) @@ -136,6 +242,8 @@ def __setstate__(self, obj): mod_globals[k] = f self.mod_globals = mod_globals + self.name_prefix = name_prefix + self.source_code = source_code # }}} @@ -146,16 +254,19 @@ class PicklableFunction: """Convenience class wrapping a function in a :class:`PicklableModule`. """ - def __init__(self, module, name): + module: PicklableModule + name: str + + def __init__(self, module: PicklableModule, name: str) -> None: self._initialize(module, name) - def _initialize(self, module, name): + def _initialize(self, module: PicklableModule, name: str) -> None: self.module = module self.name = name - self.func = module.mod_globals[name] + self._callable = cast("FunctionType", module.mod_globals[name]) # pyright: ignore[reportUnannotatedClassAttribute] - def __call__(self, *args, **kwargs): - return self.func(*args, **kwargs) + def __call__(self, *args: object, **kwargs: object) -> object: + return self._callable(*args, **kwargs) # pyright: ignore[reportAny] def __getstate__(self): return {"module": self.module, "name": self.name} diff --git a/pytools/test/test_py_codegen.py b/pytools/test/test_py_codegen.py index 5f7b280a..e53fd9b1 100644 --- a/pytools/test/test_py_codegen.py +++ b/pytools/test/test_py_codegen.py @@ -1,6 +1,8 @@ from __future__ import annotations +import pickle import sys +from typing import cast import pytest @@ -67,6 +69,56 @@ def test_function_decorators(capfd): assert out == "" # second print is not executed due to lru_cache +def test_linecache_func() -> None: + cg = codegen.PythonFunctionGenerator("f", args=()) + cg("return 42") + + func = cg.get_function() + func() + + mod_name = func.__code__.co_filename + + import linecache + + assert linecache.getlines(mod_name) == [ + "def f():\n", + " return 42\n", + ] + + assert linecache.getline(mod_name, 1) == "def f():\n" + assert linecache.getline(mod_name, 2) == " return 42\n" + + pkl = pickle.dumps(cg.get_picklable_function()) + + pf = cast("codegen.PicklableFunction", pickle.loads(pkl)) + + post_pickle_mod_name = pf._callable.__code__.co_filename + + assert post_pickle_mod_name != mod_name + assert linecache.getlines(post_pickle_mod_name) == [ + "def f():\n", + " return 42\n", + ] + + +def test_linecache_mod() -> None: + cg2 = codegen.PythonCodeGenerator() + cg2("def f():") + cg2(" return 37") + + mod = cg2.get_module() + mod["f"]() + mod_name = cast("str", mod["__code__"].co_filename) + + assert mod_name + + import linecache + assert linecache.getlines(mod_name) == [ + "def f():\n", + " return 37\n", + ] + + if __name__ == "__main__": if len(sys.argv) > 1: exec(sys.argv[1]) From 64b848212e3d856cf94eecc3736146f58c2baa2f Mon Sep 17 00:00:00 2001 From: Matthias Diener Date: Wed, 14 May 2025 16:07:29 -0500 Subject: [PATCH 2/5] Fix apt in bpr CI --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3925eccf..6851d814 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,7 +86,8 @@ jobs: curl -L -O https://tiker.net/ci-support-v0 . ./ci-support-v0 build_py_project_in_venv - sudo apt install libopenmpi-dev + sudo apt update + sudo apt -y install libopenmpi-dev pip install numpy attrs orderedsets pytest mpi4py matplotlib pip install basedpyright basedpyright From 2e86ea0a0fdb7e3050c07cb6c641498bf4e06125 Mon Sep 17 00:00:00 2001 From: Matthias Diener Date: Wed, 14 May 2025 16:45:31 -0500 Subject: [PATCH 3/5] Add NanKeyWarning, silence where appropriate add NanKeyWarning --- pytools/persistent_dict.py | 7 +++++++ pytools/test/test_persistent_dict.py | 12 +++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/pytools/persistent_dict.py b/pytools/persistent_dict.py index 0f24967c..de0b5fe4 100644 --- a/pytools/persistent_dict.py +++ b/pytools/persistent_dict.py @@ -84,6 +84,7 @@ class RecommendedHashNotFoundWarning(UserWarning): .. autoexception:: ReadOnlyEntryError .. autoexception:: CollisionWarning +.. autoexception:: NanKeyWarning .. autoclass:: KeyBuilder .. autoclass:: PersistentDict @@ -105,6 +106,11 @@ class RecommendedHashNotFoundWarning(UserWarning): # {{{ key generation +class NanKeyWarning(UserWarning): + """Warning raised when a NaN is encountered while hashing a key in + :class:`KeyBuilder`.""" + + class KeyBuilder: """A (stateless) object that computes persistent hashes of objects fed to it. Subclassing this class permits customizing the computation of hash keys. @@ -267,6 +273,7 @@ def update_for_float(self, key_hash: Hash, key: float) -> None: warn("Encountered a NaN while hashing. Since NaNs compare unequal " "to themselves, the resulting key can not be retrieved from a " "PersistentDict and will lead to a collision error on retrieval.", + NanKeyWarning, stacklevel=1) key_hash.update(key.hex().encode("utf8")) diff --git a/pytools/test/test_persistent_dict.py b/pytools/test/test_persistent_dict.py index 9b84e4e7..2cdd2a7c 100644 --- a/pytools/test/test_persistent_dict.py +++ b/pytools/test/test_persistent_dict.py @@ -999,6 +999,8 @@ def test_concurrency_threads() -> None: def test_nan_keys() -> None: # test for https://github.com/inducer/pytools/issues/287 + from pytools.persistent_dict import NanKeyWarning + with tempfile.TemporaryDirectory() as tmpdir: keyb = KeyBuilder() pdict: PersistentDict[float, int] = PersistentDict("pytools-test", @@ -1018,11 +1020,15 @@ def test_nan_keys() -> None: for nan_value in nan_values: assert nan_value != nan_value - assert keyb(nan_value) == keyb(nan_value) - pdict[nan_value] = 42 + with pytest.warns(NanKeyWarning): + assert keyb(nan_value) == keyb(nan_value) + + with pytest.warns(NanKeyWarning): + pdict[nan_value] = 42 - with (pytest.warns(CollisionWarning), + with (pytest.warns(NanKeyWarning), + pytest.warns(CollisionWarning), pytest.raises(NoSuchEntryCollisionError)): pdict[nan_value] From 0126675bb9dc440a0a25e24af8c28ad887abf75b Mon Sep 17 00:00:00 2001 From: Andreas Kloeckner Date: Mon, 26 May 2025 15:06:52 -0500 Subject: [PATCH 4/5] bpr: create a more lenient exec env for tests --- pyproject.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 1e45fcb5..fba8c35e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -117,6 +117,11 @@ reportImportCycles = "none" pythonVersion = "3.10" pythonPlatform = "All" +[[tool.basedpyright.executionEnvironments]] +root = "pytools/test" +reportUnknownArgumentType = "hint" +reportPrivateUsage = "none" + [tool.mypy] python_version = "3.10" ignore_missing_imports = true From 9722f81b8b44d57b13a9de5e75f3fb1d5db287b6 Mon Sep 17 00:00:00 2001 From: Andreas Kloeckner Date: Mon, 26 May 2025 15:09:27 -0500 Subject: [PATCH 5/5] Fire mypy --- .github/workflows/ci.yml | 15 --------------- .gitlab-ci.yml | 9 --------- 2 files changed, 24 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6851d814..876da58d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,21 +59,6 @@ jobs: curl -L -O https://gitlab.tiker.net/inducer/ci-support/raw/main/prepare-and-run-pylint.sh . ./prepare-and-run-pylint.sh "$(basename $GITHUB_REPOSITORY)" - mypy: - name: Mypy - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - name: "Main Script" - run: | - EXTRA_INSTALL="numpy" - curl -L -O https://gitlab.tiker.net/inducer/ci-support/raw/main/prepare-and-run-mypy.sh - . ./prepare-and-run-mypy.sh python3 mypy - basedpyright: runs-on: ubuntu-latest steps: diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 568089af..ece4d8db 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -50,15 +50,6 @@ Ruff: except: - tags -Mypy: - script: - - curl -L -O https://gitlab.tiker.net/inducer/ci-support/raw/main/prepare-and-run-mypy.sh - - ". ./prepare-and-run-mypy.sh python3 mypy" - tags: - - python3 - except: - - tags - Pylint: script: - EXTRA_INSTALL="numpy pymbolic orderedsets siphash24"