diff --git a/.basedpyright/baseline.json b/.basedpyright/baseline.json index f8f6a765..528606ce 100644 --- a/.basedpyright/baseline.json +++ b/.basedpyright/baseline.json @@ -14228,778 +14228,10 @@ ], "./pytools/py_codegen.py": [ { - "code": "reportUnusedImport", - "range": { - "startColumn": 4, - "endColumn": 15, - "lineCount": 1 - } - }, - { - "code": "reportUnusedImport", - "range": { - "startColumn": 4, - "endColumn": 29, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 8, - "endColumn": 18, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 25, - "endColumn": 29, - "lineCount": 1 - } - }, - { - "code": "reportMissingParameterType", - "range": { - "startColumn": 25, - "endColumn": 29, - "lineCount": 1 - } - }, - { - "code": "reportUnknownArgumentType", - "range": { - "startColumn": 39, - "endColumn": 43, - "lineCount": 1 - } - }, - { - "code": "reportUnknownArgumentType", - "range": { - "startColumn": 16, - "endColumn": 27, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 15, - "endColumn": 26, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 35, - "endColumn": 39, - "lineCount": 1 - } - }, - { - "code": "reportMissingParameterType", - "range": { - "startColumn": 35, - "endColumn": 39, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 31, - "endColumn": 46, - "lineCount": 1 - } - }, - { - "code": "reportUnknownArgumentType", - "range": { - "startColumn": 31, - "endColumn": 57, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 23, - "endColumn": 27, - "lineCount": 1 - } - }, - { - "code": "reportMissingParameterType", - "range": { - "startColumn": 23, - "endColumn": 27, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 29, - "endColumn": 33, - "lineCount": 1 - } - }, - { - "code": "reportMissingParameterType", - "range": { - "startColumn": 29, - "endColumn": 33, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 35, - "endColumn": 45, - "lineCount": 1 - } - }, - { - "code": "reportMissingParameterType", - "range": { - "startColumn": 35, - "endColumn": 45, - "lineCount": 1 - } - }, - { - "code": "reportUnannotatedClassAttribute", - "range": { - "startColumn": 13, - "endColumn": 17, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 16, - "endColumn": 25, - "lineCount": 1 - } - }, - { - "code": "reportUnknownArgumentType", - "range": { - "startColumn": 21, - "endColumn": 30, - "lineCount": 1 - } - }, - { - "code": "reportUnknownArgumentType", - "range": { - "startColumn": 34, - "endColumn": 38, - "lineCount": 1 - } - }, - { - "code": "reportUnknownArgumentType", - "range": { - "startColumn": 50, - "endColumn": 54, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 8, - "endColumn": 20, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 15, - "endColumn": 30, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 15, - "endColumn": 66, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 17, - "endColumn": 42, - "lineCount": 1 - } - }, - { - "code": "reportUnknownArgumentType", - "range": { - "startColumn": 41, - "endColumn": 50, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 4, - "endColumn": 26, - "lineCount": 1 - } - }, - { - "code": "reportUnknownArgumentType", - "range": { - "startColumn": 54, - "endColumn": 65, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 11, - "endColumn": 22, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 0, - "endColumn": 18, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 23, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportMissingParameterType", - "range": { - "startColumn": 23, - "endColumn": 34, - "lineCount": 1 - } - }, - { - "code": "reportUnannotatedClassAttribute", - "range": { - "startColumn": 13, - "endColumn": 24, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 8, - "endColumn": 20, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 12, - "endColumn": 13, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 15, - "endColumn": 16, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 20, - "endColumn": 36, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 20, - "endColumn": 42, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 15, - "endColumn": 76, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 27, - "endColumn": 30, - "lineCount": 1 - } - }, - { - "code": "reportMissingParameterType", - "range": { - "startColumn": 27, - "endColumn": 30, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 12, - "endColumn": 17, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 19, - "endColumn": 28, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 30, - "endColumn": 48, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 12, - "endColumn": 17, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 19, - "endColumn": 28, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 30, - "endColumn": 37, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 39, - "endColumn": 57, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 8, - "endColumn": 19, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 8, - "endColumn": 26, - "lineCount": 1 - } - }, - { - "code": "reportUnknownArgumentType", - "range": { - "startColumn": 27, - "endColumn": 45, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 12, - "endColumn": 13, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 15, - "endColumn": 23, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 27, - "endColumn": 40, - "lineCount": 1 - } - }, - { - "code": "reportUnknownArgumentType", - "range": { - "startColumn": 43, - "endColumn": 51, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 12, - "endColumn": 13, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 16, - "endColumn": 20, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 22, - "endColumn": 32, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 34, - "endColumn": 41, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 46, - "endColumn": 61, - "lineCount": 1 - } - }, - { - "code": "reportAny", - "range": { - "startColumn": 20, - "endColumn": 45, - "lineCount": 1 - } - }, - { - "code": "reportUnknownArgumentType", - "range": { - "startColumn": 34, - "endColumn": 44, - "lineCount": 1 - } - }, - { - "code": "reportUnknownArgumentType", - "range": { - "startColumn": 47, - "endColumn": 58, - "lineCount": 1 - } - }, - { - "code": "reportUnknownArgumentType", - "range": { - "startColumn": 65, - "endColumn": 69, - "lineCount": 1 - } - }, - { - "code": "reportUnknownArgumentType", - "range": { - "startColumn": 28, - "endColumn": 35, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 8, - "endColumn": 24, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 23, - "endColumn": 29, - "lineCount": 1 - } - }, - { - "code": "reportMissingParameterType", - "range": { - "startColumn": 23, - "endColumn": 29, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 31, - "endColumn": 35, - "lineCount": 1 - } - }, - { - "code": "reportMissingParameterType", - "range": { - "startColumn": 31, - "endColumn": 35, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 8, - "endColumn": 24, - "lineCount": 1 - } - }, - { - "code": "reportUnknownArgumentType", - "range": { - "startColumn": 25, - "endColumn": 31, - "lineCount": 1 - } - }, - { - "code": "reportUnknownArgumentType", - "range": { - "startColumn": 33, - "endColumn": 37, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 26, - "endColumn": 32, - "lineCount": 1 - } - }, - { - "code": "reportMissingParameterType", - "range": { - "startColumn": 26, - "endColumn": 32, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 34, - "endColumn": 38, - "lineCount": 1 - } - }, - { - "code": "reportMissingParameterType", - "range": { - "startColumn": 34, - "endColumn": 38, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 8, - "endColumn": 19, - "lineCount": 1 - } - }, - { - "code": "reportUnannotatedClassAttribute", - "range": { - "startColumn": 13, - "endColumn": 19, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 8, - "endColumn": 17, - "lineCount": 1 - } - }, - { - "code": "reportUnannotatedClassAttribute", - "range": { - "startColumn": 13, - "endColumn": 17, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 8, - "endColumn": 17, - "lineCount": 1 - } - }, - { - "code": "reportUnannotatedClassAttribute", - "range": { - "startColumn": 13, - "endColumn": 17, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", + "code": "reportAny", "range": { "startColumn": 20, - "endColumn": 38, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 8, - "endColumn": 16, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 24, - "endColumn": 28, - "lineCount": 1 - } - }, - { - "code": "reportMissingParameterType", - "range": { - "startColumn": 24, - "endColumn": 28, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 32, - "endColumn": 38, - "lineCount": 1 - } - }, - { - "code": "reportMissingParameterType", - "range": { - "startColumn": 32, - "endColumn": 38, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 15, - "endColumn": 24, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 15, - "endColumn": 41, - "lineCount": 1 - } - }, - { - "code": "reportUnknownParameterType", - "range": { - "startColumn": 8, - "endColumn": 20, - "lineCount": 1 - } - }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 15, - "endColumn": 57, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 26, - "endColumn": 37, - "lineCount": 1 - } - }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 47, - "endColumn": 56, + "endColumn": 45, "lineCount": 1 } }, @@ -15019,14 +14251,6 @@ "lineCount": 1 } }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 8, - "endColumn": 24, - "lineCount": 1 - } - }, { "code": "reportUnknownArgumentType", "range": { @@ -17875,14 +17099,6 @@ "lineCount": 1 } }, - { - "code": "reportPrivateUsage", - "range": { - "startColumn": 40, - "endColumn": 53, - "lineCount": 1 - } - }, { "code": "reportUnknownParameterType", "range": { @@ -18341,14 +17557,6 @@ "lineCount": 1 } }, - { - "code": "reportUnknownMemberType", - "range": { - "startColumn": 36, - "endColumn": 59, - "lineCount": 1 - } - }, { "code": "reportAny", "range": { @@ -18389,14 +17597,6 @@ "lineCount": 1 } }, - { - "code": "reportUnknownVariableType", - "range": { - "startColumn": 4, - "endColumn": 5, - "lineCount": 1 - } - }, { "code": "reportUnknownVariableType", "range": { diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3925eccf..876da58d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,21 +59,6 @@ jobs: curl -L -O https://gitlab.tiker.net/inducer/ci-support/raw/main/prepare-and-run-pylint.sh . ./prepare-and-run-pylint.sh "$(basename $GITHUB_REPOSITORY)" - mypy: - name: Mypy - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - name: "Main Script" - run: | - EXTRA_INSTALL="numpy" - curl -L -O https://gitlab.tiker.net/inducer/ci-support/raw/main/prepare-and-run-mypy.sh - . ./prepare-and-run-mypy.sh python3 mypy - basedpyright: runs-on: ubuntu-latest steps: @@ -86,7 +71,8 @@ jobs: curl -L -O https://tiker.net/ci-support-v0 . ./ci-support-v0 build_py_project_in_venv - sudo apt install libopenmpi-dev + sudo apt update + sudo apt -y install libopenmpi-dev pip install numpy attrs orderedsets pytest mpi4py matplotlib pip install basedpyright basedpyright diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 568089af..ece4d8db 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -50,15 +50,6 @@ Ruff: except: - tags -Mypy: - script: - - curl -L -O https://gitlab.tiker.net/inducer/ci-support/raw/main/prepare-and-run-mypy.sh - - ". ./prepare-and-run-mypy.sh python3 mypy" - tags: - - python3 - except: - - tags - Pylint: script: - EXTRA_INSTALL="numpy pymbolic orderedsets siphash24" diff --git a/pyproject.toml b/pyproject.toml index 1e45fcb5..fba8c35e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -117,6 +117,11 @@ reportImportCycles = "none" pythonVersion = "3.10" pythonPlatform = "All" +[[tool.basedpyright.executionEnvironments]] +root = "pytools/test" +reportUnknownArgumentType = "hint" +reportPrivateUsage = "none" + [tool.mypy] python_version = "3.10" ignore_missing_imports = true diff --git a/pytools/__init__.py b/pytools/__init__.py index 2c9b56fa..6e28fc6d 100644 --- a/pytools/__init__.py +++ b/pytools/__init__.py @@ -2226,13 +2226,15 @@ def generate_unique_names(prefix): def generate_numbered_unique_names( - prefix: str, num: int | None = None) -> Iterable[tuple[int, str]]: + prefix: str, + num: int | None = None, + suffix: str = "") -> Iterable[tuple[int, str]]: if num is None: - yield (0, prefix) + yield (0, prefix + suffix) num = 0 while True: - name = f"{prefix}_{num}" + name = f"{prefix}_{num}{suffix}" num += 1 yield (num, name) @@ -2254,19 +2256,22 @@ class UniqueNameGenerator: """ def __init__(self, existing_names: Collection[str] | None = None, - forced_prefix: str = ""): + forced_prefix: str = "", + forced_suffix: str = "") -> None: """ Create a new :class:`UniqueNameGenerator`. :arg existing_names: a :class:`set` of existing names that will be skipped when generating new names. :arg forced_prefix: all generated :class:`str` have this prefix. + :arg forced_suffix: all generated :class:`str` have this suffix. """ if existing_names is None: existing_names = set() self.existing_names = set(existing_names) self.forced_prefix = forced_prefix + self.forced_suffix: str = forced_suffix self.prefix_to_counter: dict[str, int] = {} def is_name_conflicting(self, name: str) -> bool: @@ -2326,7 +2331,8 @@ def __call__(self, based_on: str = "id") -> str: # }}} - for counter, var_name in generate_numbered_unique_names(based_on, counter): # noqa: B020,B007 + for counter, var_name in generate_numbered_unique_names( # noqa: B020,B007 + based_on, counter, self.forced_suffix): if not self.is_name_conflicting(var_name): break diff --git a/pytools/persistent_dict.py b/pytools/persistent_dict.py index 0f24967c..de0b5fe4 100644 --- a/pytools/persistent_dict.py +++ b/pytools/persistent_dict.py @@ -84,6 +84,7 @@ class RecommendedHashNotFoundWarning(UserWarning): .. autoexception:: ReadOnlyEntryError .. autoexception:: CollisionWarning +.. autoexception:: NanKeyWarning .. autoclass:: KeyBuilder .. autoclass:: PersistentDict @@ -105,6 +106,11 @@ class RecommendedHashNotFoundWarning(UserWarning): # {{{ key generation +class NanKeyWarning(UserWarning): + """Warning raised when a NaN is encountered while hashing a key in + :class:`KeyBuilder`.""" + + class KeyBuilder: """A (stateless) object that computes persistent hashes of objects fed to it. Subclassing this class permits customizing the computation of hash keys. @@ -267,6 +273,7 @@ def update_for_float(self, key_hash: Hash, key: float) -> None: warn("Encountered a NaN while hashing. Since NaNs compare unequal " "to themselves, the resulting key can not be retrieved from a " "PersistentDict and will lead to a collision error on retrieval.", + NanKeyWarning, stacklevel=1) key_hash.update(key.hex().encode("utf8")) diff --git a/pytools/py_codegen.py b/pytools/py_codegen.py index 01943d89..3f799ce3 100644 --- a/pytools/py_codegen.py +++ b/pytools/py_codegen.py @@ -23,79 +23,162 @@ THE SOFTWARE. """ + import marshal +from dataclasses import dataclass, field from importlib.util import MAGIC_NUMBER as BYTECODE_VERSION from types import FunctionType, ModuleType +from typing import TYPE_CHECKING, Any, Literal, TypeAlias, cast + + +if TYPE_CHECKING: + from collections.abc import Callable, Iterable -from pytools.codegen import ( # noqa: F401 +from pytools.codegen import ( CodeGenerator as CodeGeneratorBase, Indentation, remove_common_indentation, ) -class PythonCodeGenerator(CodeGeneratorBase): - def get_module(self, name=None): - if name is None: - name = "" +__all__ = ( + "ExistingLineCacheWarning", + "Indentation", + "PicklableFunction", + "PicklableModule", + "PythonCodeGenerator", + "PythonFunctionGenerator", + "remove_common_indentation", +) + + +class ExistingLineCacheWarning(Warning): + """Warning for overwriting existing generated code in the linecache.""" + + +def _linecache_unique_name(name_prefix: str, source_text: str | None) -> str: + import linecache + + if source_text is not None: + from siphash24 import siphash13 # pyright: ignore[reportUnknownVariableType] + src_digest = cast("str", siphash13(source_text.encode()).hexdigest()) # pyright: ignore[reportUnknownMemberType] + + name_prefix = f"{name_prefix}-{src_digest}" + + from pytools import UniqueNameGenerator + name_gen = UniqueNameGenerator( + existing_names=linecache.cache.keys(), + forced_prefix=" dict[str, Any]: + if name_prefix is None: + name_prefix = "module" + + if name is None: + name = _linecache_unique_name(name_prefix, source_text) + + result_dict: dict[str, Any] = {} + + # {{{ insert into linecache + + import linecache - result_dict = {} - source_text = self.get() - exec(compile( - source_text.rstrip()+"\n", name, "exec"), - result_dict) - result_dict["_MODULE_SOURCE_CODE"] = source_text - return result_dict + if name in linecache.cache: + from warnings import warn + warn(f"Overwriting existing generated code in linecache: '{name}'.", + ExistingLineCacheWarning, + stacklevel=2) - def get_picklable_module(self, name=None): - return PicklableModule(self.get_module(name=name)) + linecache.cache[name] = (len(source_text), None, + [e+"\n" for e in source_text.split("\n")], name) + + # }}} + + code_obj = compile( + source_text.rstrip()+"\n", name, "exec") + result_dict["__code__"] = code_obj + exec(code_obj, result_dict) + + return result_dict + + +class PythonCodeGenerator(CodeGeneratorBase): + def get_module(self, name: str | None = None, + *, + name_prefix: str | None = None, + ) -> dict[str, Any]: + return _make_module(self.get(), name=name, name_prefix=name_prefix) + + def get_picklable_module(self, + name: str | None = None, + name_prefix: str | None = None + ) -> PicklableModule: + return PicklableModule(self.get_module(name=name, name_prefix=name_prefix)) class PythonFunctionGenerator(PythonCodeGenerator): - def __init__(self, name, args, decorators=None): - PythonCodeGenerator.__init__(self) + name: str + + def __init__(self, name: str, args: Iterable[str], + decorators: Iterable[str] = ()) -> None: + super().__init__() self.name = name - if decorators: - for decorator in decorators: - self(decorator) + for decorator in decorators: + self(decorator) self("def {}({}):".format(name, ", ".join(args))) self.indent() - @property - def _gen_filename(self): - return f"" + def get_function(self) -> Callable[..., Any]: + return self.get_module(name_prefix=self.name)[self.name] # pyright: ignore[reportAny] - def get_function(self): - return self.get_module(name=self._gen_filename)[self.name] - - def get_picklable_function(self): - module = self.get_picklable_module(name=self._gen_filename) - return PicklableFunction(module, self.name) + def get_picklable_function(self) -> PicklableFunction: + return PicklableFunction( + self.get_picklable_module(name_prefix=self.name), self.name) # {{{ pickling of binaries for generated code -def _get_empty_module_dict(): - result_dict = {} - exec(compile("", "", "exec"), result_dict) +def _get_empty_module_dict(filename: str | None = None) -> dict[str, Any]: + if filename is None: + filename = "" + + result_dict: dict[str, Any] = {} + code_obj = compile("", filename, "exec") + result_dict["__code__"] = code_obj + exec(code_obj, result_dict) return result_dict _empty_module_dict = _get_empty_module_dict() +_FunctionsType: TypeAlias = dict[str, tuple[str, bytes, tuple[object, ...] | None]] +_ModulesType: TypeAlias = dict[str, str] + + +@dataclass class PicklableModule: - def __init__(self, mod_globals): - self.mod_globals = mod_globals + mod_globals: dict[str, Any] + name_prefix: str | None = field(kw_only=True, default=None) + source_code: str | None = field(kw_only=True, default=None) def __getstate__(self): - nondefault_globals = {} - functions = {} - modules = {} + functions: _FunctionsType = {} + modules: _ModulesType = {} + nondefault_globals: dict[str, object] = {} - for k, v in self.mod_globals.items(): + for k, v in self.mod_globals.items(): # pyright: ignore[reportAny] if isinstance(v, FunctionType): functions[k] = ( v.__name__, @@ -106,14 +189,29 @@ def __getstate__(self): elif k not in _empty_module_dict: nondefault_globals[k] = v - return (1, BYTECODE_VERSION, functions, modules, nondefault_globals) - - def __setstate__(self, obj): + return (2, BYTECODE_VERSION, functions, modules, nondefault_globals, + self.name_prefix, self.source_code) + + def __setstate__(self, obj: ( + tuple[Literal[0], bytes, _FunctionsType, dict[str, object]] + | tuple[Literal[1], bytes, _FunctionsType, _ModulesType, + dict[str, object]] + | tuple[Literal[2], bytes, _FunctionsType, _ModulesType, + dict[str, object], str | None, str | None] + ) + ): if obj[0] == 0: magic, functions, nondefault_globals = obj[1:] modules = {} + name_prefix: str | None = None + source_code: str | None = None elif obj[0] == 1: magic, functions, modules, nondefault_globals = obj[1:] + name_prefix = None + source_code = None + elif obj[0] == 2: + magic, functions, modules, nondefault_globals, name_prefix, source_code = \ + obj[1:] else: raise ValueError("unknown version of PicklableModule") @@ -122,9 +220,17 @@ def __setstate__(self, obj): "cannot unpickle function binary: incorrect magic value " f"(got: {magic!r}, expected: {BYTECODE_VERSION!r})") - mod_globals = _empty_module_dict.copy() + unique_filename = _linecache_unique_name( + name_prefix if name_prefix else "module", source_code) + mod_globals = _get_empty_module_dict(unique_filename) mod_globals.update(nondefault_globals) + import linecache + if source_code: + linecache.cache[unique_filename] = (len(source_code), None, + [e+"\n" for e in source_code.split("\n")], + unique_filename) + from importlib import import_module for k, mod_name in modules.items(): mod_globals[k] = import_module(mod_name) @@ -136,6 +242,8 @@ def __setstate__(self, obj): mod_globals[k] = f self.mod_globals = mod_globals + self.name_prefix = name_prefix + self.source_code = source_code # }}} @@ -146,16 +254,19 @@ class PicklableFunction: """Convenience class wrapping a function in a :class:`PicklableModule`. """ - def __init__(self, module, name): + module: PicklableModule + name: str + + def __init__(self, module: PicklableModule, name: str) -> None: self._initialize(module, name) - def _initialize(self, module, name): + def _initialize(self, module: PicklableModule, name: str) -> None: self.module = module self.name = name - self.func = module.mod_globals[name] + self._callable = cast("FunctionType", module.mod_globals[name]) # pyright: ignore[reportUnannotatedClassAttribute] - def __call__(self, *args, **kwargs): - return self.func(*args, **kwargs) + def __call__(self, *args: object, **kwargs: object) -> object: + return self._callable(*args, **kwargs) # pyright: ignore[reportAny] def __getstate__(self): return {"module": self.module, "name": self.name} diff --git a/pytools/test/test_persistent_dict.py b/pytools/test/test_persistent_dict.py index 9b84e4e7..2cdd2a7c 100644 --- a/pytools/test/test_persistent_dict.py +++ b/pytools/test/test_persistent_dict.py @@ -999,6 +999,8 @@ def test_concurrency_threads() -> None: def test_nan_keys() -> None: # test for https://github.com/inducer/pytools/issues/287 + from pytools.persistent_dict import NanKeyWarning + with tempfile.TemporaryDirectory() as tmpdir: keyb = KeyBuilder() pdict: PersistentDict[float, int] = PersistentDict("pytools-test", @@ -1018,11 +1020,15 @@ def test_nan_keys() -> None: for nan_value in nan_values: assert nan_value != nan_value - assert keyb(nan_value) == keyb(nan_value) - pdict[nan_value] = 42 + with pytest.warns(NanKeyWarning): + assert keyb(nan_value) == keyb(nan_value) + + with pytest.warns(NanKeyWarning): + pdict[nan_value] = 42 - with (pytest.warns(CollisionWarning), + with (pytest.warns(NanKeyWarning), + pytest.warns(CollisionWarning), pytest.raises(NoSuchEntryCollisionError)): pdict[nan_value] diff --git a/pytools/test/test_py_codegen.py b/pytools/test/test_py_codegen.py index 5f7b280a..e53fd9b1 100644 --- a/pytools/test/test_py_codegen.py +++ b/pytools/test/test_py_codegen.py @@ -1,6 +1,8 @@ from __future__ import annotations +import pickle import sys +from typing import cast import pytest @@ -67,6 +69,56 @@ def test_function_decorators(capfd): assert out == "" # second print is not executed due to lru_cache +def test_linecache_func() -> None: + cg = codegen.PythonFunctionGenerator("f", args=()) + cg("return 42") + + func = cg.get_function() + func() + + mod_name = func.__code__.co_filename + + import linecache + + assert linecache.getlines(mod_name) == [ + "def f():\n", + " return 42\n", + ] + + assert linecache.getline(mod_name, 1) == "def f():\n" + assert linecache.getline(mod_name, 2) == " return 42\n" + + pkl = pickle.dumps(cg.get_picklable_function()) + + pf = cast("codegen.PicklableFunction", pickle.loads(pkl)) + + post_pickle_mod_name = pf._callable.__code__.co_filename + + assert post_pickle_mod_name != mod_name + assert linecache.getlines(post_pickle_mod_name) == [ + "def f():\n", + " return 42\n", + ] + + +def test_linecache_mod() -> None: + cg2 = codegen.PythonCodeGenerator() + cg2("def f():") + cg2(" return 37") + + mod = cg2.get_module() + mod["f"]() + mod_name = cast("str", mod["__code__"].co_filename) + + assert mod_name + + import linecache + assert linecache.getlines(mod_name) == [ + "def f():\n", + " return 37\n", + ] + + if __name__ == "__main__": if len(sys.argv) > 1: exec(sys.argv[1])