From fbebca289d811669fc1980e3a135325b8542a846 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Wed, 12 Nov 2025 09:59:48 +0500 Subject: [PATCH 001/638] GH-116946: eliminate the need for the GC in the `_thread.lock` and `_thread.RLock` (#141268) --- Modules/_threadmodule.c | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index cc8277c5783858..0e22c7bd386be1 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -41,6 +41,7 @@ typedef struct { typedef struct { PyObject_HEAD PyMutex lock; + PyObject *weakreflist; /* List of weak references */ } lockobject; #define lockobject_CAST(op) ((lockobject *)(op)) @@ -48,6 +49,7 @@ typedef struct { typedef struct { PyObject_HEAD _PyRecursiveMutex lock; + PyObject *weakreflist; /* List of weak references */ } rlockobject; #define rlockobject_CAST(op) ((rlockobject *)(op)) @@ -767,7 +769,6 @@ static PyType_Spec ThreadHandle_Type_spec = { static void lock_dealloc(PyObject *self) { - PyObject_GC_UnTrack(self); PyObject_ClearWeakRefs(self); PyTypeObject *tp = Py_TYPE(self); tp->tp_free(self); @@ -999,6 +1000,10 @@ lock_new_impl(PyTypeObject *type) return (PyObject *)self; } +static PyMemberDef lock_members[] = { + {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(lockobject, weakreflist), Py_READONLY}, + {NULL} +}; static PyMethodDef lock_methods[] = { _THREAD_LOCK_ACQUIRE_LOCK_METHODDEF @@ -1034,8 +1039,8 @@ static PyType_Slot lock_type_slots[] = { {Py_tp_dealloc, lock_dealloc}, {Py_tp_repr, lock_repr}, {Py_tp_doc, (void *)lock_doc}, + {Py_tp_members, lock_members}, {Py_tp_methods, lock_methods}, - {Py_tp_traverse, _PyObject_VisitType}, {Py_tp_new, lock_new}, {0, 0} }; @@ -1043,8 +1048,7 @@ static PyType_Slot lock_type_slots[] = { static PyType_Spec lock_type_spec = { .name = "_thread.lock", .basicsize = sizeof(lockobject), - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | - Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_MANAGED_WEAKREF), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE), .slots = lock_type_slots, }; @@ -1059,7 +1063,6 @@ rlock_locked_impl(rlockobject *self) static void rlock_dealloc(PyObject *self) { - PyObject_GC_UnTrack(self); PyObject_ClearWeakRefs(self); PyTypeObject *tp = Py_TYPE(self); tp->tp_free(self); @@ -1319,6 +1322,11 @@ _thread_RLock__at_fork_reinit_impl(rlockobject *self) #endif /* HAVE_FORK */ +static PyMemberDef rlock_members[] = { + {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(rlockobject, weakreflist), Py_READONLY}, + {NULL} +}; + static PyMethodDef rlock_methods[] = { _THREAD_RLOCK_ACQUIRE_METHODDEF _THREAD_RLOCK_RELEASE_METHODDEF @@ -1339,10 +1347,10 @@ static PyMethodDef rlock_methods[] = { static PyType_Slot rlock_type_slots[] = { {Py_tp_dealloc, rlock_dealloc}, {Py_tp_repr, rlock_repr}, + {Py_tp_members, rlock_members}, {Py_tp_methods, rlock_methods}, {Py_tp_alloc, PyType_GenericAlloc}, {Py_tp_new, rlock_new}, - {Py_tp_traverse, _PyObject_VisitType}, {0, 0}, }; @@ -1350,7 +1358,7 @@ static PyType_Spec rlock_type_spec = { .name = "_thread.RLock", .basicsize = sizeof(rlockobject), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | - Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_MANAGED_WEAKREF), + Py_TPFLAGS_IMMUTABLETYPE), .slots = rlock_type_slots, }; From ef474cfafbdf3aa383fb1334a7ab95cef9834ced Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Wed, 12 Nov 2025 10:47:38 +0530 Subject: [PATCH 002/638] gh-103847: fix cancellation safety of `asyncio.create_subprocess_exec` (#140805) --- Lib/asyncio/base_subprocess.py | 11 +++++ Lib/test/test_asyncio/test_subprocess.py | 40 ++++++++++++++++++- ...-10-31-13-57-55.gh-issue-103847.VM7TnW.rst | 1 + 3 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-10-31-13-57-55.gh-issue-103847.VM7TnW.rst diff --git a/Lib/asyncio/base_subprocess.py b/Lib/asyncio/base_subprocess.py index d40af422e614c1..321a4e5d5d18fb 100644 --- a/Lib/asyncio/base_subprocess.py +++ b/Lib/asyncio/base_subprocess.py @@ -26,6 +26,7 @@ def __init__(self, loop, protocol, args, shell, self._pending_calls = collections.deque() self._pipes = {} self._finished = False + self._pipes_connected = False if stdin == subprocess.PIPE: self._pipes[0] = None @@ -213,6 +214,7 @@ async def _connect_pipes(self, waiter): else: if waiter is not None and not waiter.cancelled(): waiter.set_result(None) + self._pipes_connected = True def _call(self, cb, *data): if self._pending_calls is not None: @@ -256,6 +258,15 @@ def _try_finish(self): assert not self._finished if self._returncode is None: return + if not self._pipes_connected: + # self._pipes_connected can be False if not all pipes were connected + # because either the process failed to start or the self._connect_pipes task + # got cancelled. In this broken state we consider all pipes disconnected and + # to avoid hanging forever in self._wait as otherwise _exit_waiters + # would never be woken up, we wake them up here. + for waiter in self._exit_waiters: + if not waiter.cancelled(): + waiter.set_result(self._returncode) if all(p is not None and p.disconnected for p in self._pipes.values()): self._finished = True diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index 3a17c169c34f12..bf301740741ae7 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -11,7 +11,7 @@ from asyncio import subprocess from test.test_asyncio import utils as test_utils from test import support -from test.support import os_helper +from test.support import os_helper, warnings_helper, gc_collect if not support.has_subprocess_support: raise unittest.SkipTest("test module requires subprocess") @@ -879,6 +879,44 @@ async def main(): self.loop.run_until_complete(main()) + @warnings_helper.ignore_warnings(category=ResourceWarning) + def test_subprocess_read_pipe_cancelled(self): + async def main(): + loop = asyncio.get_running_loop() + loop.connect_read_pipe = mock.AsyncMock(side_effect=asyncio.CancelledError) + with self.assertRaises(asyncio.CancelledError): + await asyncio.create_subprocess_exec(*PROGRAM_BLOCKED, stderr=asyncio.subprocess.PIPE) + + asyncio.run(main()) + gc_collect() + + @warnings_helper.ignore_warnings(category=ResourceWarning) + def test_subprocess_write_pipe_cancelled(self): + async def main(): + loop = asyncio.get_running_loop() + loop.connect_write_pipe = mock.AsyncMock(side_effect=asyncio.CancelledError) + with self.assertRaises(asyncio.CancelledError): + await asyncio.create_subprocess_exec(*PROGRAM_BLOCKED, stdin=asyncio.subprocess.PIPE) + + asyncio.run(main()) + gc_collect() + + @warnings_helper.ignore_warnings(category=ResourceWarning) + def test_subprocess_read_write_pipe_cancelled(self): + async def main(): + loop = asyncio.get_running_loop() + loop.connect_read_pipe = mock.AsyncMock(side_effect=asyncio.CancelledError) + loop.connect_write_pipe = mock.AsyncMock(side_effect=asyncio.CancelledError) + with self.assertRaises(asyncio.CancelledError): + await asyncio.create_subprocess_exec( + *PROGRAM_BLOCKED, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + + asyncio.run(main()) + gc_collect() if sys.platform != 'win32': # Unix diff --git a/Misc/NEWS.d/next/Library/2025-10-31-13-57-55.gh-issue-103847.VM7TnW.rst b/Misc/NEWS.d/next/Library/2025-10-31-13-57-55.gh-issue-103847.VM7TnW.rst new file mode 100644 index 00000000000000..e14af7d97083d6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-31-13-57-55.gh-issue-103847.VM7TnW.rst @@ -0,0 +1 @@ +Fix hang when cancelling process created by :func:`asyncio.create_subprocess_exec` or :func:`asyncio.create_subprocess_shell`. Patch by Kumar Aditya. From f1b7961ccfa050e9c80622fff1b3cdada46f9aab Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Wed, 12 Nov 2025 12:51:43 +0530 Subject: [PATCH 003/638] GH-116946: revert eliminate the need for the GC in the `_thread.lock` and `_thread.RLock` (#141448) Revert "GH-116946: eliminate the need for the GC in the `_thread.lock` and `_thread.RLock` (#141268)" This reverts commit fbebca289d811669fc1980e3a135325b8542a846. --- Modules/_threadmodule.c | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 0e22c7bd386be1..cc8277c5783858 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -41,7 +41,6 @@ typedef struct { typedef struct { PyObject_HEAD PyMutex lock; - PyObject *weakreflist; /* List of weak references */ } lockobject; #define lockobject_CAST(op) ((lockobject *)(op)) @@ -49,7 +48,6 @@ typedef struct { typedef struct { PyObject_HEAD _PyRecursiveMutex lock; - PyObject *weakreflist; /* List of weak references */ } rlockobject; #define rlockobject_CAST(op) ((rlockobject *)(op)) @@ -769,6 +767,7 @@ static PyType_Spec ThreadHandle_Type_spec = { static void lock_dealloc(PyObject *self) { + PyObject_GC_UnTrack(self); PyObject_ClearWeakRefs(self); PyTypeObject *tp = Py_TYPE(self); tp->tp_free(self); @@ -1000,10 +999,6 @@ lock_new_impl(PyTypeObject *type) return (PyObject *)self; } -static PyMemberDef lock_members[] = { - {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(lockobject, weakreflist), Py_READONLY}, - {NULL} -}; static PyMethodDef lock_methods[] = { _THREAD_LOCK_ACQUIRE_LOCK_METHODDEF @@ -1039,8 +1034,8 @@ static PyType_Slot lock_type_slots[] = { {Py_tp_dealloc, lock_dealloc}, {Py_tp_repr, lock_repr}, {Py_tp_doc, (void *)lock_doc}, - {Py_tp_members, lock_members}, {Py_tp_methods, lock_methods}, + {Py_tp_traverse, _PyObject_VisitType}, {Py_tp_new, lock_new}, {0, 0} }; @@ -1048,7 +1043,8 @@ static PyType_Slot lock_type_slots[] = { static PyType_Spec lock_type_spec = { .name = "_thread.lock", .basicsize = sizeof(lockobject), - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_MANAGED_WEAKREF), .slots = lock_type_slots, }; @@ -1063,6 +1059,7 @@ rlock_locked_impl(rlockobject *self) static void rlock_dealloc(PyObject *self) { + PyObject_GC_UnTrack(self); PyObject_ClearWeakRefs(self); PyTypeObject *tp = Py_TYPE(self); tp->tp_free(self); @@ -1322,11 +1319,6 @@ _thread_RLock__at_fork_reinit_impl(rlockobject *self) #endif /* HAVE_FORK */ -static PyMemberDef rlock_members[] = { - {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(rlockobject, weakreflist), Py_READONLY}, - {NULL} -}; - static PyMethodDef rlock_methods[] = { _THREAD_RLOCK_ACQUIRE_METHODDEF _THREAD_RLOCK_RELEASE_METHODDEF @@ -1347,10 +1339,10 @@ static PyMethodDef rlock_methods[] = { static PyType_Slot rlock_type_slots[] = { {Py_tp_dealloc, rlock_dealloc}, {Py_tp_repr, rlock_repr}, - {Py_tp_members, rlock_members}, {Py_tp_methods, rlock_methods}, {Py_tp_alloc, PyType_GenericAlloc}, {Py_tp_new, rlock_new}, + {Py_tp_traverse, _PyObject_VisitType}, {0, 0}, }; @@ -1358,7 +1350,7 @@ static PyType_Spec rlock_type_spec = { .name = "_thread.RLock", .basicsize = sizeof(rlockobject), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | - Py_TPFLAGS_IMMUTABLETYPE), + Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_MANAGED_WEAKREF), .slots = rlock_type_slots, }; From 35908265b09ac39b67116bfdfe8a053be09e6d8f Mon Sep 17 00:00:00 2001 From: Mark Byrne <31762852+mbyrnepr2@users.noreply.github.com> Date: Wed, 12 Nov 2025 09:20:55 +0100 Subject: [PATCH 004/638] gh-75593: Add support of bytes and path-like paths in wave.open() (GH-140951) --- Doc/library/wave.rst | 9 ++++++-- Lib/test/test_wave.py | 22 +++++++++++++++++++ Lib/wave.py | 5 +++-- ...5-11-04-12-16-13.gh-issue-75593.EFVhKR.rst | 1 + 4 files changed, 33 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-04-12-16-13.gh-issue-75593.EFVhKR.rst diff --git a/Doc/library/wave.rst b/Doc/library/wave.rst index a3f5bfd5e2f99c..7ff2c97992c4e3 100644 --- a/Doc/library/wave.rst +++ b/Doc/library/wave.rst @@ -25,8 +25,9 @@ The :mod:`wave` module defines the following function and exception: .. function:: open(file, mode=None) - If *file* is a string, open the file by that name, otherwise treat it as a - file-like object. *mode* can be: + If *file* is a string, a :term:`path-like object` or a + :term:`bytes-like object` open the file by that name, otherwise treat it as + a file-like object. *mode* can be: ``'rb'`` Read only mode. @@ -52,6 +53,10 @@ The :mod:`wave` module defines the following function and exception: .. versionchanged:: 3.4 Added support for unseekable files. + .. versionchanged:: 3.15 + Added support for :term:`path-like objects ` + and :term:`bytes-like objects `. + .. exception:: Error An error raised when something is impossible because it violates the WAV diff --git a/Lib/test/test_wave.py b/Lib/test/test_wave.py index 226b1aa84bd73c..4c21f16553775c 100644 --- a/Lib/test/test_wave.py +++ b/Lib/test/test_wave.py @@ -1,9 +1,11 @@ import unittest from test import audiotests from test import support +from test.support.os_helper import FakePath import io import os import struct +import tempfile import sys import wave @@ -206,5 +208,25 @@ def test_open_in_write_raises(self): self.assertIsNone(cm.unraisable) +class WaveOpen(unittest.TestCase): + def test_open_pathlike(self): + """It is possible to use `wave.read` and `wave.write` with a path-like object""" + with tempfile.NamedTemporaryFile(delete_on_close=False) as fp: + cases = ( + FakePath(fp.name), + FakePath(os.fsencode(fp.name)), + os.fsencode(fp.name), + ) + for fake_path in cases: + with self.subTest(fake_path): + with wave.open(fake_path, 'wb') as f: + f.setnchannels(1) + f.setsampwidth(2) + f.setframerate(44100) + + with wave.open(fake_path, 'rb') as f: + pass + + if __name__ == '__main__': unittest.main() diff --git a/Lib/wave.py b/Lib/wave.py index 5af745e2217ec3..056bd6aab7ffa3 100644 --- a/Lib/wave.py +++ b/Lib/wave.py @@ -69,6 +69,7 @@ from collections import namedtuple import builtins +import os import struct import sys @@ -274,7 +275,7 @@ def initfp(self, file): def __init__(self, f): self._i_opened_the_file = None - if isinstance(f, str): + if isinstance(f, (bytes, str, os.PathLike)): f = builtins.open(f, 'rb') self._i_opened_the_file = f # else, assume it is an open file object already @@ -431,7 +432,7 @@ class Wave_write: def __init__(self, f): self._i_opened_the_file = None - if isinstance(f, str): + if isinstance(f, (bytes, str, os.PathLike)): f = builtins.open(f, 'wb') self._i_opened_the_file = f try: diff --git a/Misc/NEWS.d/next/Library/2025-11-04-12-16-13.gh-issue-75593.EFVhKR.rst b/Misc/NEWS.d/next/Library/2025-11-04-12-16-13.gh-issue-75593.EFVhKR.rst new file mode 100644 index 00000000000000..9a31af9c110454 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-04-12-16-13.gh-issue-75593.EFVhKR.rst @@ -0,0 +1 @@ +Add support of :term:`path-like objects ` and :term:`bytes-like objects ` in :func:`wave.open`. From 909f76dab91f028edd2ae7bd589d3975996de9e1 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 12 Nov 2025 09:42:56 +0100 Subject: [PATCH 005/638] gh-141376: Rename _AsyncioDebug to _Py_AsyncioDebug (GH-141391) --- Modules/_asynciomodule.c | 4 ++-- Tools/c-analyzer/cpython/ignored.tsv | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 1f58b1fb3506c6..9b2b7011244d77 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -119,7 +119,7 @@ typedef struct _Py_AsyncioModuleDebugOffsets { } asyncio_thread_state; } Py_AsyncioModuleDebugOffsets; -GENERATE_DEBUG_SECTION(AsyncioDebug, Py_AsyncioModuleDebugOffsets _AsyncioDebug) +GENERATE_DEBUG_SECTION(AsyncioDebug, Py_AsyncioModuleDebugOffsets _Py_AsyncioDebug) = {.asyncio_task_object = { .size = sizeof(TaskObj), .task_name = offsetof(TaskObj, task_name), @@ -4338,7 +4338,7 @@ module_init(asyncio_state *state) goto fail; } - state->debug_offsets = &_AsyncioDebug; + state->debug_offsets = &_Py_AsyncioDebug; Py_DECREF(module); return 0; diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 8b73189fb07dc5..11a3cd794ff4d7 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -56,7 +56,7 @@ Python/pyhash.c - _Py_HashSecret - Python/parking_lot.c - buckets - ## data needed for introspecting asyncio state from debuggers and profilers -Modules/_asynciomodule.c - _AsyncioDebug - +Modules/_asynciomodule.c - _Py_AsyncioDebug - ################################## From 6f988b08d122e44848e89c04ad1e10c25d072cc7 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Wed, 12 Nov 2025 01:37:48 -0800 Subject: [PATCH 006/638] gh-85524: Raise "UnsupportedOperation" on FileIO.readall (#141214) io.UnsupportedOperation is a subclass of OSError and recommended by io.IOBase for this case; matches other read methods on io.FileIO. --- Lib/test/test_io/test_general.py | 1 + .../2025-11-07-12-25-46.gh-issue-85524.9SWFIC.rst | 3 +++ Modules/_io/clinic/fileio.c.h | 14 +++++++++----- Modules/_io/fileio.c | 13 ++++++++++--- 4 files changed, 23 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-07-12-25-46.gh-issue-85524.9SWFIC.rst diff --git a/Lib/test/test_io/test_general.py b/Lib/test/test_io/test_general.py index a1cdd6876c2892..f0677b01ea5ce1 100644 --- a/Lib/test/test_io/test_general.py +++ b/Lib/test/test_io/test_general.py @@ -125,6 +125,7 @@ def test_invalid_operations(self): self.assertRaises(exc, fp.readline) with self.open(os_helper.TESTFN, "wb", buffering=0) as fp: self.assertRaises(exc, fp.read) + self.assertRaises(exc, fp.readall) self.assertRaises(exc, fp.readline) with self.open(os_helper.TESTFN, "rb", buffering=0) as fp: self.assertRaises(exc, fp.write, b"blah") diff --git a/Misc/NEWS.d/next/Library/2025-11-07-12-25-46.gh-issue-85524.9SWFIC.rst b/Misc/NEWS.d/next/Library/2025-11-07-12-25-46.gh-issue-85524.9SWFIC.rst new file mode 100644 index 00000000000000..3e4fd1a5897b04 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-07-12-25-46.gh-issue-85524.9SWFIC.rst @@ -0,0 +1,3 @@ +Update ``io.FileIO.readall``, an implementation of :meth:`io.RawIOBase.readall`, +to follow :class:`io.IOBase` guidelines and raise :exc:`io.UnsupportedOperation` +when a file is in "w" mode rather than :exc:`OSError` diff --git a/Modules/_io/clinic/fileio.c.h b/Modules/_io/clinic/fileio.c.h index 04870b1c890361..96c31ce8d6f415 100644 --- a/Modules/_io/clinic/fileio.c.h +++ b/Modules/_io/clinic/fileio.c.h @@ -277,15 +277,19 @@ PyDoc_STRVAR(_io_FileIO_readall__doc__, "data is available (EAGAIN is returned before bytes are read) returns None."); #define _IO_FILEIO_READALL_METHODDEF \ - {"readall", (PyCFunction)_io_FileIO_readall, METH_NOARGS, _io_FileIO_readall__doc__}, + {"readall", _PyCFunction_CAST(_io_FileIO_readall), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io_FileIO_readall__doc__}, static PyObject * -_io_FileIO_readall_impl(fileio *self); +_io_FileIO_readall_impl(fileio *self, PyTypeObject *cls); static PyObject * -_io_FileIO_readall(PyObject *self, PyObject *Py_UNUSED(ignored)) +_io_FileIO_readall(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - return _io_FileIO_readall_impl((fileio *)self); + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { + PyErr_SetString(PyExc_TypeError, "readall() takes no arguments"); + return NULL; + } + return _io_FileIO_readall_impl((fileio *)self, cls); } PyDoc_STRVAR(_io_FileIO_read__doc__, @@ -543,4 +547,4 @@ _io_FileIO_isatty(PyObject *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO_FILEIO_TRUNCATE_METHODDEF #define _IO_FILEIO_TRUNCATE_METHODDEF #endif /* !defined(_IO_FILEIO_TRUNCATE_METHODDEF) */ -/*[clinic end generated code: output=1902fac9e39358aa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2e48f3df2f189170 input=a9049054013a1b77]*/ diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 2544ff4ea91ec8..5d7741fdd830a5 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -728,6 +728,9 @@ new_buffersize(fileio *self, size_t currentsize) @permit_long_docstring_body _io.FileIO.readall + cls: defining_class + / + Read all data from the file, returned as bytes. Reads until either there is an error or read() returns size 0 (indicates EOF). @@ -738,8 +741,8 @@ data is available (EAGAIN is returned before bytes are read) returns None. [clinic start generated code]*/ static PyObject * -_io_FileIO_readall_impl(fileio *self) -/*[clinic end generated code: output=faa0292b213b4022 input=10d8b2ec403302dc]*/ +_io_FileIO_readall_impl(fileio *self, PyTypeObject *cls) +/*[clinic end generated code: output=d546737ec895c462 input=cecda40bf9961299]*/ { Py_off_t pos, end; PyBytesWriter *writer; @@ -750,6 +753,10 @@ _io_FileIO_readall_impl(fileio *self) if (self->fd < 0) { return err_closed(); } + if (!self->readable) { + _PyIO_State *state = get_io_state_by_cls(cls); + return err_mode(state, "reading"); + } if (self->stat_atopen != NULL && self->stat_atopen->st_size < _PY_READ_MAX) { end = (Py_off_t)self->stat_atopen->st_size; @@ -873,7 +880,7 @@ _io_FileIO_read_impl(fileio *self, PyTypeObject *cls, Py_ssize_t size) } if (size < 0) - return _io_FileIO_readall_impl(self); + return _io_FileIO_readall_impl(self, cls); if (size > _PY_READ_MAX) { size = _PY_READ_MAX; From 20f53df07d42c495a08c73a3d54b8dd9098a62f0 Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Wed, 12 Nov 2025 12:50:44 +0300 Subject: [PATCH 007/638] gh-141370: document undefined behavior of Py_ABS() (GH-141439) --- Doc/c-api/intro.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index 6e1a9dcb35543b..c76cc2f70ecccf 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -121,6 +121,10 @@ complete listing. Return the absolute value of ``x``. + If the result cannot be represented (for example, if ``x`` has + :c:macro:`!INT_MIN` value for :c:expr:`int` type), the behavior is + undefined. + .. versionadded:: 3.3 .. c:macro:: Py_ALWAYS_INLINE From 7d54374f9c7d91e0ef90c4ad84baf10073cf1d8a Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Wed, 12 Nov 2025 01:57:05 -0800 Subject: [PATCH 008/638] gh-141311: Avoid assertion in BytesIO.readinto() (GH-141333) Fix error in assertion which causes failure if pos is equal to PY_SSIZE_T_MAX. Fix undefined behavior in read() and readinto() if pos is larger that the size of the underlying buffer. --- Lib/test/test_io/test_memoryio.py | 14 ++++++++++++++ ...025-11-09-18-55-13.gh-issue-141311.qZ3swc.rst | 2 ++ Modules/_io/bytesio.c | 16 +++++++++++++--- 3 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-09-18-55-13.gh-issue-141311.qZ3swc.rst diff --git a/Lib/test/test_io/test_memoryio.py b/Lib/test/test_io/test_memoryio.py index 63998a86c45b53..bb023735e21398 100644 --- a/Lib/test/test_io/test_memoryio.py +++ b/Lib/test/test_io/test_memoryio.py @@ -54,6 +54,12 @@ def testSeek(self): self.assertEqual(buf[3:], bytesIo.read()) self.assertRaises(TypeError, bytesIo.seek, 0.0) + self.assertEqual(sys.maxsize, bytesIo.seek(sys.maxsize)) + self.assertEqual(self.EOF, bytesIo.read(4)) + + self.assertEqual(sys.maxsize - 2, bytesIo.seek(sys.maxsize - 2)) + self.assertEqual(self.EOF, bytesIo.read(4)) + def testTell(self): buf = self.buftype("1234567890") bytesIo = self.ioclass(buf) @@ -552,6 +558,14 @@ def test_relative_seek(self): memio.seek(1, 1) self.assertEqual(memio.read(), buf[1:]) + def test_issue141311(self): + memio = self.ioclass() + # Seek allows PY_SSIZE_T_MAX, read should handle that. + # Past end of buffer read should always return 0 (EOF). + self.assertEqual(sys.maxsize, memio.seek(sys.maxsize)) + buf = bytearray(2) + self.assertEqual(0, memio.readinto(buf)) + def test_unicode(self): memio = self.ioclass() diff --git a/Misc/NEWS.d/next/Library/2025-11-09-18-55-13.gh-issue-141311.qZ3swc.rst b/Misc/NEWS.d/next/Library/2025-11-09-18-55-13.gh-issue-141311.qZ3swc.rst new file mode 100644 index 00000000000000..bb425ce5df309d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-09-18-55-13.gh-issue-141311.qZ3swc.rst @@ -0,0 +1,2 @@ +Fix assertion failure in :func:`!io.BytesIO.readinto` and undefined behavior +arising when read position is above capcity in :class:`io.BytesIO`. diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index d6bfb93177c9ee..96611823ab6b45 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -436,6 +436,13 @@ read_bytes_lock_held(bytesio *self, Py_ssize_t size) return Py_NewRef(self->buf); } + /* gh-141311: Avoid undefined behavior when self->pos (limit PY_SSIZE_T_MAX) + is beyond the size of self->buf. Assert above validates size is always in + bounds. When self->pos is out of bounds calling code sets size to 0. */ + if (size == 0) { + return PyBytes_FromStringAndSize(NULL, 0); + } + output = PyBytes_AS_STRING(self->buf) + self->pos; self->pos += size; return PyBytes_FromStringAndSize(output, size); @@ -609,11 +616,14 @@ _io_BytesIO_readinto_impl(bytesio *self, Py_buffer *buffer) n = self->string_size - self->pos; if (len > n) { len = n; - if (len < 0) - len = 0; + if (len < 0) { + /* gh-141311: Avoid undefined behavior when self->pos (limit + PY_SSIZE_T_MAX) points beyond the size of self->buf. */ + return PyLong_FromSsize_t(0); + } } - assert(self->pos + len < PY_SSIZE_T_MAX); + assert(self->pos + len <= PY_SSIZE_T_MAX); assert(len >= 0); memcpy(buffer->buf, PyBytes_AS_STRING(self->buf) + self->pos, len); self->pos += len; From 23d85a2a3fb029172ea15c6e596f64f8c2868ed3 Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Wed, 12 Nov 2025 13:06:29 +0300 Subject: [PATCH 009/638] gh-141042: fix sNaN's packing for mixed floating-point formats (#141107) --- Lib/test/test_capi/test_float.py | 54 +++++++++++++++---- ...-11-06-06-28-14.gh-issue-141042.brOioJ.rst | 3 ++ Objects/floatobject.c | 16 ++++-- 3 files changed, 59 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-11-06-06-28-14.gh-issue-141042.brOioJ.rst diff --git a/Lib/test/test_capi/test_float.py b/Lib/test/test_capi/test_float.py index 983b991b4f163d..df7017e6436a69 100644 --- a/Lib/test/test_capi/test_float.py +++ b/Lib/test/test_capi/test_float.py @@ -29,6 +29,23 @@ NAN = float("nan") +def make_nan(size, sign, quiet, payload=None): + if size == 8: + payload_mask = 0x7ffffffffffff + i = (sign << 63) + (0x7ff << 52) + (quiet << 51) + elif size == 4: + payload_mask = 0x3fffff + i = (sign << 31) + (0xff << 23) + (quiet << 22) + elif size == 2: + payload_mask = 0x1ff + i = (sign << 15) + (0x1f << 10) + (quiet << 9) + else: + raise ValueError("size must be either 2, 4, or 8") + if payload is None: + payload = random.randint(not quiet, payload_mask) + return i + payload + + class CAPIFloatTest(unittest.TestCase): def test_check(self): # Test PyFloat_Check() @@ -202,16 +219,7 @@ def test_pack_unpack_roundtrip_for_nans(self): # HP PA RISC uses 0 for quiet, see: # https://en.wikipedia.org/wiki/NaN#Encoding signaling = 1 - quiet = int(not signaling) - if size == 8: - payload = random.randint(signaling, 0x7ffffffffffff) - i = (sign << 63) + (0x7ff << 52) + (quiet << 51) + payload - elif size == 4: - payload = random.randint(signaling, 0x3fffff) - i = (sign << 31) + (0xff << 23) + (quiet << 22) + payload - elif size == 2: - payload = random.randint(signaling, 0x1ff) - i = (sign << 15) + (0x1f << 10) + (quiet << 9) + payload + i = make_nan(size, sign, not signaling) data = bytes.fromhex(f'{i:x}') for endian in (BIG_ENDIAN, LITTLE_ENDIAN): with self.subTest(data=data, size=size, endian=endian): @@ -221,6 +229,32 @@ def test_pack_unpack_roundtrip_for_nans(self): self.assertTrue(math.isnan(value)) self.assertEqual(data1, data2) + @unittest.skipUnless(HAVE_IEEE_754, "requires IEEE 754") + @unittest.skipUnless(sys.maxsize != 2147483647, "requires 64-bit mode") + def test_pack_unpack_nans_for_different_formats(self): + pack = _testcapi.float_pack + unpack = _testcapi.float_unpack + + for endian in (BIG_ENDIAN, LITTLE_ENDIAN): + with self.subTest(endian=endian): + byteorder = "big" if endian == BIG_ENDIAN else "little" + + # Convert sNaN to qNaN, if payload got truncated + data = make_nan(8, 0, False, 0x80001).to_bytes(8, byteorder) + snan_low = unpack(data, endian) + qnan4 = make_nan(4, 0, True, 0).to_bytes(4, byteorder) + qnan2 = make_nan(2, 0, True, 0).to_bytes(2, byteorder) + self.assertEqual(pack(4, snan_low, endian), qnan4) + self.assertEqual(pack(2, snan_low, endian), qnan2) + + # Preserve NaN type, if payload not truncated + data = make_nan(8, 0, False, 0x80000000001).to_bytes(8, byteorder) + snan_high = unpack(data, endian) + snan4 = make_nan(4, 0, False, 16384).to_bytes(4, byteorder) + snan2 = make_nan(2, 0, False, 2).to_bytes(2, byteorder) + self.assertEqual(pack(4, snan_high, endian), snan4) + self.assertEqual(pack(2, snan_high, endian), snan2) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/C_API/2025-11-06-06-28-14.gh-issue-141042.brOioJ.rst b/Misc/NEWS.d/next/C_API/2025-11-06-06-28-14.gh-issue-141042.brOioJ.rst new file mode 100644 index 00000000000000..22a1aa1f405318 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-11-06-06-28-14.gh-issue-141042.brOioJ.rst @@ -0,0 +1,3 @@ +Make qNaN in :c:func:`PyFloat_Pack2` and :c:func:`PyFloat_Pack4`, if while +conversion to a narrower precision floating-point format --- the remaining +after truncation payload will be zero. Patch by Sergey B Kirpichev. diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 1fefb12803ec19..ef613efe4e7f44 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -2030,6 +2030,10 @@ PyFloat_Pack2(double x, char *data, int le) memcpy(&v, &x, sizeof(v)); v &= 0xffc0000000000ULL; bits = (unsigned short)(v >> 42); /* NaN's type & payload */ + /* set qNaN if no payload */ + if (!bits) { + bits |= (1<<9); + } } else { sign = (x < 0.0); @@ -2202,16 +2206,16 @@ PyFloat_Pack4(double x, char *data, int le) if ((v & (1ULL << 51)) == 0) { uint32_t u32; memcpy(&u32, &y, 4); - u32 &= ~(1 << 22); /* make sNaN */ + /* if have payload, make sNaN */ + if (u32 & 0x3fffff) { + u32 &= ~(1 << 22); + } memcpy(&y, &u32, 4); } #else uint32_t u32; memcpy(&u32, &y, 4); - if ((v & (1ULL << 51)) == 0) { - u32 &= ~(1 << 22); - } /* Workaround RISC-V: "If a NaN value is converted to a * different floating-point type, the result is the * canonical NaN of the new type". The canonical NaN here @@ -2222,6 +2226,10 @@ PyFloat_Pack4(double x, char *data, int le) /* add payload */ u32 -= (u32 & 0x3fffff); u32 += (uint32_t)((v & 0x7ffffffffffffULL) >> 29); + /* if have payload, make sNaN */ + if ((v & (1ULL << 51)) == 0 && (u32 & 0x3fffff)) { + u32 &= ~(1 << 22); + } memcpy(&y, &u32, 4); #endif From 70748bdbea872a84dd8eadad9b48c73e218d2e1f Mon Sep 17 00:00:00 2001 From: Jacob Austin Lincoln <99031153+lincolnj1@users.noreply.github.com> Date: Wed, 12 Nov 2025 02:07:21 -0800 Subject: [PATCH 010/638] gh-131116: Fix inspect.getdoc() to work with cached_property objects (GH-131165) --- Doc/library/inspect.rst | 3 ++ Lib/inspect.py | 6 +++ Lib/test/test_inspect/inspect_fodder3.py | 39 +++++++++++++++++++ Lib/test/test_inspect/test_inspect.py | 20 ++++++++++ ...-03-12-18-57-10.gh-issue-131116.uTpwXZ.rst | 2 + 5 files changed, 70 insertions(+) create mode 100644 Lib/test/test_inspect/inspect_fodder3.py create mode 100644 Misc/NEWS.d/next/Library/2025-03-12-18-57-10.gh-issue-131116.uTpwXZ.rst diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index aff53b78c4a774..13a352cbdb2cdc 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -639,6 +639,9 @@ Retrieving source code .. versionchanged:: next Added parameters *inherit_class_doc* and *fallback_to_class_doc*. + Documentation strings on :class:`~functools.cached_property` + objects are now inherited if not overriden. + .. function:: getcomments(object) diff --git a/Lib/inspect.py b/Lib/inspect.py index bb17848b444b67..8e7511b3af015f 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -747,6 +747,12 @@ def _finddoc(obj, *, search_in_class=True): cls = _findclass(obj.fget) if cls is None or getattr(cls, name) is not obj: return None + # Should be tested before ismethoddescriptor() + elif isinstance(obj, functools.cached_property): + name = obj.attrname + cls = _findclass(obj.func) + if cls is None or getattr(cls, name) is not obj: + return None elif ismethoddescriptor(obj) or isdatadescriptor(obj): name = obj.__name__ cls = obj.__objclass__ diff --git a/Lib/test/test_inspect/inspect_fodder3.py b/Lib/test/test_inspect/inspect_fodder3.py new file mode 100644 index 00000000000000..ea2481edf938c2 --- /dev/null +++ b/Lib/test/test_inspect/inspect_fodder3.py @@ -0,0 +1,39 @@ +from functools import cached_property + +# docstring in parent, inherited in child +class ParentInheritDoc: + @cached_property + def foo(self): + """docstring for foo defined in parent""" + +class ChildInheritDoc(ParentInheritDoc): + pass + +class ChildInheritDefineDoc(ParentInheritDoc): + @cached_property + def foo(self): + pass + +# Redefine foo as something other than cached_property +class ChildPropertyFoo(ParentInheritDoc): + @property + def foo(self): + """docstring for the property foo""" + +class ChildMethodFoo(ParentInheritDoc): + def foo(self): + """docstring for the method foo""" + +# docstring in child but not parent +class ParentNoDoc: + @cached_property + def foo(self): + pass + +class ChildNoDoc(ParentNoDoc): + pass + +class ChildDefineDoc(ParentNoDoc): + @cached_property + def foo(self): + """docstring for foo defined in child""" diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 24fd4a2fa626d4..dd3b7d9c5b4b5b 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -46,6 +46,7 @@ from test.test_inspect import inspect_fodder as mod from test.test_inspect import inspect_fodder2 as mod2 +from test.test_inspect import inspect_fodder3 as mod3 from test.test_inspect import inspect_stringized_annotations from test.test_inspect import inspect_deferred_annotations @@ -714,6 +715,25 @@ class B(A): b.__doc__ = 'Instance' self.assertEqual(inspect.getdoc(b, fallback_to_class_doc=False), 'Instance') + def test_getdoc_inherited_cached_property(self): + doc = inspect.getdoc(mod3.ParentInheritDoc.foo) + self.assertEqual(doc, 'docstring for foo defined in parent') + self.assertEqual(inspect.getdoc(mod3.ChildInheritDoc.foo), doc) + self.assertEqual(inspect.getdoc(mod3.ChildInheritDefineDoc.foo), doc) + + def test_getdoc_redefine_cached_property_as_other(self): + self.assertEqual(inspect.getdoc(mod3.ChildPropertyFoo.foo), + 'docstring for the property foo') + self.assertEqual(inspect.getdoc(mod3.ChildMethodFoo.foo), + 'docstring for the method foo') + + def test_getdoc_define_cached_property(self): + self.assertEqual(inspect.getdoc(mod3.ChildDefineDoc.foo), + 'docstring for foo defined in child') + + def test_getdoc_nodoc_inherited(self): + self.assertIsNone(inspect.getdoc(mod3.ChildNoDoc.foo)) + @unittest.skipIf(MISSING_C_DOCSTRINGS, "test requires docstrings") def test_finddoc(self): finddoc = inspect._finddoc diff --git a/Misc/NEWS.d/next/Library/2025-03-12-18-57-10.gh-issue-131116.uTpwXZ.rst b/Misc/NEWS.d/next/Library/2025-03-12-18-57-10.gh-issue-131116.uTpwXZ.rst new file mode 100644 index 00000000000000..f5e60ab6e8c4cb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-03-12-18-57-10.gh-issue-131116.uTpwXZ.rst @@ -0,0 +1,2 @@ +:func:`inspect.getdoc` now correctly returns an inherited docstring on +:class:`~functools.cached_property` objects if none is given in a subclass. From c6f3dd6a506a9bb1808c070e5ef5cf345a3bedc8 Mon Sep 17 00:00:00 2001 From: Rani Pinchuk <33353578+rani-pinchuk@users.noreply.github.com> Date: Wed, 12 Nov 2025 13:35:01 +0100 Subject: [PATCH 011/638] gh-98896: resource_tracker: use json&base64 to allow arbitrary shared memory names (GH-138473) --- Lib/multiprocessing/resource_tracker.py | 60 ++++++++++++++++--- Lib/test/_test_multiprocessing.py | 43 +++++++++++++ ...5-09-03-20-18-39.gh-issue-98896.tjez89.rst | 2 + 3 files changed, 97 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-09-03-20-18-39.gh-issue-98896.tjez89.rst diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index 38fcaed48fa9fb..b0f9099f4a59f3 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -15,6 +15,7 @@ # this resource tracker process, "killall python" would probably leave unlinked # resources. +import base64 import os import signal import sys @@ -22,6 +23,8 @@ import warnings from collections import deque +import json + from . import spawn from . import util @@ -196,6 +199,17 @@ def _launch(self): finally: os.close(r) + def _make_probe_message(self): + """Return a JSON-encoded probe message.""" + return ( + json.dumps( + {"cmd": "PROBE", "rtype": "noop"}, + ensure_ascii=True, + separators=(",", ":"), + ) + + "\n" + ).encode("ascii") + def _ensure_running_and_write(self, msg=None): with self._lock: if self._lock._recursion_count() > 1: @@ -207,7 +221,7 @@ def _ensure_running_and_write(self, msg=None): if self._fd is not None: # resource tracker was launched before, is it still running? if msg is None: - to_send = b'PROBE:0:noop\n' + to_send = self._make_probe_message() else: to_send = msg try: @@ -234,7 +248,7 @@ def _check_alive(self): try: # We cannot use send here as it calls ensure_running, creating # a cycle. - os.write(self._fd, b'PROBE:0:noop\n') + os.write(self._fd, self._make_probe_message()) except OSError: return False else: @@ -253,11 +267,25 @@ def _write(self, msg): assert nbytes == len(msg), f"{nbytes=} != {len(msg)=}" def _send(self, cmd, name, rtype): - msg = f"{cmd}:{name}:{rtype}\n".encode("ascii") - if len(msg) > 512: - # posix guarantees that writes to a pipe of less than PIPE_BUF - # bytes are atomic, and that PIPE_BUF >= 512 - raise ValueError('msg too long') + # POSIX guarantees that writes to a pipe of less than PIPE_BUF (512 on Linux) + # bytes are atomic. Therefore, we want the message to be shorter than 512 bytes. + # POSIX shm_open() and sem_open() require the name, including its leading slash, + # to be at most NAME_MAX bytes (255 on Linux) + # With json.dump(..., ensure_ascii=True) every non-ASCII byte becomes a 6-char + # escape like \uDC80. + # As we want the overall message to be kept atomic and therefore smaller than 512, + # we encode encode the raw name bytes with URL-safe Base64 - so a 255 long name + # will not exceed 340 bytes. + b = name.encode('utf-8', 'surrogateescape') + if len(b) > 255: + raise ValueError('shared memory name too long (max 255 bytes)') + b64 = base64.urlsafe_b64encode(b).decode('ascii') + + payload = {"cmd": cmd, "rtype": rtype, "base64_name": b64} + msg = (json.dumps(payload, ensure_ascii=True, separators=(",", ":")) + "\n").encode("ascii") + + # The entire JSON message is guaranteed < PIPE_BUF (512 bytes) by construction. + assert len(msg) <= 512, f"internal error: message too long ({len(msg)} bytes)" self._ensure_running_and_write(msg) @@ -290,7 +318,23 @@ def main(fd): with open(fd, 'rb') as f: for line in f: try: - cmd, name, rtype = line.strip().decode('ascii').split(':') + try: + obj = json.loads(line.decode('ascii')) + except Exception as e: + raise ValueError("malformed resource_tracker message: %r" % (line,)) from e + + cmd = obj["cmd"] + rtype = obj["rtype"] + b64 = obj.get("base64_name", "") + + if not isinstance(cmd, str) or not isinstance(rtype, str) or not isinstance(b64, str): + raise ValueError("malformed resource_tracker fields: %r" % (obj,)) + + try: + name = base64.urlsafe_b64decode(b64).decode('utf-8', 'surrogateescape') + except ValueError as e: + raise ValueError("malformed resource_tracker base64_name: %r" % (b64,)) from e + cleanup_func = _CLEANUP_FUNCS.get(rtype, None) if cleanup_func is None: raise ValueError( diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 850744e47d0e0b..0f9c5c222250ae 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -7364,3 +7364,46 @@ def test_forkpty(self): res = assert_python_failure("-c", code, PYTHONWARNINGS='error') self.assertIn(b'DeprecationWarning', res.err) self.assertIn(b'is multi-threaded, use of forkpty() may lead to deadlocks in the child', res.err) + +@unittest.skipUnless(HAS_SHMEM, "requires multiprocessing.shared_memory") +class TestSharedMemoryNames(unittest.TestCase): + def test_that_shared_memory_name_with_colons_has_no_resource_tracker_errors(self): + # Test script that creates and cleans up shared memory with colon in name + test_script = textwrap.dedent(""" + import sys + from multiprocessing import shared_memory + import time + + # Test various patterns of colons in names + test_names = [ + "a:b", + "a:b:c", + "test:name:with:many:colons", + ":starts:with:colon", + "ends:with:colon:", + "::double::colons::", + "name\\nwithnewline", + "name-with-trailing-newline\\n", + "\\nname-starts-with-newline", + "colons:and\\nnewlines:mix", + "multi\\nline\\nname", + ] + + for name in test_names: + try: + shm = shared_memory.SharedMemory(create=True, size=100, name=name) + shm.buf[:5] = b'hello' # Write something to the shared memory + shm.close() + shm.unlink() + + except Exception as e: + print(f"Error with name '{name}': {e}", file=sys.stderr) + sys.exit(1) + + print("SUCCESS") + """) + + rc, out, err = assert_python_ok("-c", test_script) + self.assertIn(b"SUCCESS", out) + self.assertNotIn(b"traceback", err.lower(), err) + self.assertNotIn(b"resource_tracker.py", err, err) diff --git a/Misc/NEWS.d/next/Library/2025-09-03-20-18-39.gh-issue-98896.tjez89.rst b/Misc/NEWS.d/next/Library/2025-09-03-20-18-39.gh-issue-98896.tjez89.rst new file mode 100644 index 00000000000000..6831499c0afb43 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-09-03-20-18-39.gh-issue-98896.tjez89.rst @@ -0,0 +1,2 @@ +Fix a failure in multiprocessing resource_tracker when SharedMemory names contain colons. +Patch by Rani Pinchuk. From e2026731f5680022bd016b8b5ca5841c82e9574c Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Wed, 12 Nov 2025 15:44:49 +0300 Subject: [PATCH 012/638] gh-141004: soft-deprecate Py_INFINITY macro (#141033) Co-authored-by: Victor Stinner --- Doc/c-api/conversion.rst | 2 +- Doc/c-api/float.rst | 7 +++++-- Doc/whatsnew/3.14.rst | 2 +- Doc/whatsnew/3.15.rst | 4 ++++ Include/floatobject.h | 16 +++++++-------- Include/internal/pycore_pymath.h | 6 +++--- Include/pymath.h | 3 ++- ...-11-05-04-38-16.gh-issue-141004.rJL43P.rst | 1 + Modules/cmathmodule.c | 6 +++--- Modules/mathmodule.c | 20 +++++++++---------- Objects/complexobject.c | 8 ++++---- Objects/floatobject.c | 2 +- Python/pystrtod.c | 4 ++-- 13 files changed, 45 insertions(+), 36 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-11-05-04-38-16.gh-issue-141004.rJL43P.rst diff --git a/Doc/c-api/conversion.rst b/Doc/c-api/conversion.rst index 533e5460da8952..a18bbf4e0e37d7 100644 --- a/Doc/c-api/conversion.rst +++ b/Doc/c-api/conversion.rst @@ -105,7 +105,7 @@ The following functions provide locale-independent string to number conversions. If ``s`` represents a value that is too large to store in a float (for example, ``"1e500"`` is such a string on many platforms) then - if ``overflow_exception`` is ``NULL`` return ``Py_INFINITY`` (with + if ``overflow_exception`` is ``NULL`` return :c:macro:`!INFINITY` (with an appropriate sign) and don't set any exception. Otherwise, ``overflow_exception`` must point to a Python exception object; raise that exception and return ``-1.0``. In both cases, set diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index eae4792af7d299..b6020533a2b9d9 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -83,8 +83,11 @@ Floating-Point Objects This macro expands a to constant expression of type :c:expr:`double`, that represents the positive infinity. - On most platforms, this is equivalent to the :c:macro:`!INFINITY` macro from - the C11 standard ```` header. + It is equivalent to the :c:macro:`!INFINITY` macro from the C11 standard + ```` header. + + .. deprecated:: 3.15 + The macro is soft deprecated. .. c:macro:: Py_NAN diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 1a2fbda0c4ce81..9459b73bcb502f 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -3045,7 +3045,7 @@ Deprecated C APIs ----------------- * The :c:macro:`!Py_HUGE_VAL` macro is now :term:`soft deprecated`. - Use :c:macro:`!Py_INFINITY` instead. + Use :c:macro:`!INFINITY` instead. (Contributed by Sergey B Kirpichev in :gh:`120026`.) * The :c:macro:`!Py_IS_NAN`, :c:macro:`!Py_IS_INFINITY`, diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index c543b6e6c2a779..f0fd49c9033bc1 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -1095,6 +1095,10 @@ Deprecated C APIs since 3.15 and will be removed in 3.17. (Contributed by Nikita Sobolev in :gh:`136355`.) +* :c:macro:`!Py_INFINITY` macro is :term:`soft deprecated`, + use the C11 standard ```` :c:macro:`!INFINITY` instead. + (Contributed by Sergey B Kirpichev in :gh:`141004`.) + * :c:macro:`!Py_MATH_El` and :c:macro:`!Py_MATH_PIl` are deprecated since 3.15 and will be removed in 3.20. (Contributed by Sergey B Kirpichev in :gh:`141004`.) diff --git a/Include/floatobject.h b/Include/floatobject.h index 4d24a76edd5de1..814337b070ab50 100644 --- a/Include/floatobject.h +++ b/Include/floatobject.h @@ -18,14 +18,14 @@ PyAPI_DATA(PyTypeObject) PyFloat_Type; #define Py_RETURN_NAN return PyFloat_FromDouble(Py_NAN) -#define Py_RETURN_INF(sign) \ - do { \ - if (copysign(1., sign) == 1.) { \ - return PyFloat_FromDouble(Py_INFINITY); \ - } \ - else { \ - return PyFloat_FromDouble(-Py_INFINITY); \ - } \ +#define Py_RETURN_INF(sign) \ + do { \ + if (copysign(1., sign) == 1.) { \ + return PyFloat_FromDouble(INFINITY); \ + } \ + else { \ + return PyFloat_FromDouble(-INFINITY); \ + } \ } while(0) PyAPI_FUNC(double) PyFloat_GetMax(void); diff --git a/Include/internal/pycore_pymath.h b/Include/internal/pycore_pymath.h index eea8996ba68ca0..4fcac3aab8bf51 100644 --- a/Include/internal/pycore_pymath.h +++ b/Include/internal/pycore_pymath.h @@ -33,7 +33,7 @@ extern "C" { static inline void _Py_ADJUST_ERANGE1(double x) { if (errno == 0) { - if (x == Py_INFINITY || x == -Py_INFINITY) { + if (x == INFINITY || x == -INFINITY) { errno = ERANGE; } } @@ -44,8 +44,8 @@ static inline void _Py_ADJUST_ERANGE1(double x) static inline void _Py_ADJUST_ERANGE2(double x, double y) { - if (x == Py_INFINITY || x == -Py_INFINITY || - y == Py_INFINITY || y == -Py_INFINITY) + if (x == INFINITY || x == -INFINITY || + y == INFINITY || y == -INFINITY) { if (errno == 0) { errno = ERANGE; diff --git a/Include/pymath.h b/Include/pymath.h index 0f9f0f3b2990fe..7cfe441365df78 100644 --- a/Include/pymath.h +++ b/Include/pymath.h @@ -45,13 +45,14 @@ #define Py_IS_FINITE(X) isfinite(X) // Py_INFINITY: Value that evaluates to a positive double infinity. +// Soft deprecated since Python 3.15, use INFINITY instead. #ifndef Py_INFINITY # define Py_INFINITY ((double)INFINITY) #endif /* Py_HUGE_VAL should always be the same as Py_INFINITY. But historically * this was not reliable and Python did not require IEEE floats and C99 - * conformity. The macro was soft deprecated in Python 3.14, use Py_INFINITY instead. + * conformity. The macro was soft deprecated in Python 3.14, use INFINITY instead. */ #ifndef Py_HUGE_VAL # define Py_HUGE_VAL HUGE_VAL diff --git a/Misc/NEWS.d/next/C_API/2025-11-05-04-38-16.gh-issue-141004.rJL43P.rst b/Misc/NEWS.d/next/C_API/2025-11-05-04-38-16.gh-issue-141004.rJL43P.rst new file mode 100644 index 00000000000000..a054f8eda6fb0b --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-11-05-04-38-16.gh-issue-141004.rJL43P.rst @@ -0,0 +1 @@ +The :c:macro:`!Py_INFINITY` macro is :term:`soft deprecated`. diff --git a/Modules/cmathmodule.c b/Modules/cmathmodule.c index a4ea5557a6a415..aee3e4f343d8be 100644 --- a/Modules/cmathmodule.c +++ b/Modules/cmathmodule.c @@ -150,7 +150,7 @@ special_type(double d) #define P14 0.25*Py_MATH_PI #define P12 0.5*Py_MATH_PI #define P34 0.75*Py_MATH_PI -#define INF Py_INFINITY +#define INF INFINITY #define N Py_NAN #define U -9.5426319407711027e33 /* unlikely value, used as placeholder */ @@ -1186,11 +1186,11 @@ cmath_exec(PyObject *mod) if (PyModule_Add(mod, "tau", PyFloat_FromDouble(Py_MATH_TAU)) < 0) { return -1; } - if (PyModule_Add(mod, "inf", PyFloat_FromDouble(Py_INFINITY)) < 0) { + if (PyModule_Add(mod, "inf", PyFloat_FromDouble(INFINITY)) < 0) { return -1; } - Py_complex infj = {0.0, Py_INFINITY}; + Py_complex infj = {0.0, INFINITY}; if (PyModule_Add(mod, "infj", PyComplex_FromCComplex(infj)) < 0) { return -1; } diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index de1886451eda8f..11c46c987e146a 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -395,7 +395,7 @@ m_tgamma(double x) if (x == 0.0) { errno = EDOM; /* tgamma(+-0.0) = +-inf, divide-by-zero */ - return copysign(Py_INFINITY, x); + return copysign(INFINITY, x); } /* integer arguments */ @@ -426,7 +426,7 @@ m_tgamma(double x) } else { errno = ERANGE; - return Py_INFINITY; + return INFINITY; } } @@ -490,14 +490,14 @@ m_lgamma(double x) if (isnan(x)) return x; /* lgamma(nan) = nan */ else - return Py_INFINITY; /* lgamma(+-inf) = +inf */ + return INFINITY; /* lgamma(+-inf) = +inf */ } /* integer arguments */ if (x == floor(x) && x <= 2.0) { if (x <= 0.0) { errno = EDOM; /* lgamma(n) = inf, divide-by-zero for */ - return Py_INFINITY; /* integers n <= 0 */ + return INFINITY; /* integers n <= 0 */ } else { return 0.0; /* lgamma(1) = lgamma(2) = 0.0 */ @@ -633,7 +633,7 @@ m_log(double x) return log(x); errno = EDOM; if (x == 0.0) - return -Py_INFINITY; /* log(0) = -inf */ + return -INFINITY; /* log(0) = -inf */ else return Py_NAN; /* log(-ve) = nan */ } @@ -676,7 +676,7 @@ m_log2(double x) } else if (x == 0.0) { errno = EDOM; - return -Py_INFINITY; /* log2(0) = -inf, divide-by-zero */ + return -INFINITY; /* log2(0) = -inf, divide-by-zero */ } else { errno = EDOM; @@ -692,7 +692,7 @@ m_log10(double x) return log10(x); errno = EDOM; if (x == 0.0) - return -Py_INFINITY; /* log10(0) = -inf */ + return -INFINITY; /* log10(0) = -inf */ else return Py_NAN; /* log10(-ve) = nan */ } @@ -1500,7 +1500,7 @@ math_ldexp_impl(PyObject *module, double x, PyObject *i) errno = 0; } else if (exp > INT_MAX) { /* overflow */ - r = copysign(Py_INFINITY, x); + r = copysign(INFINITY, x); errno = ERANGE; } else if (exp < INT_MIN) { /* underflow to +-0 */ @@ -2983,7 +2983,7 @@ math_ulp_impl(PyObject *module, double x) if (isinf(x)) { return x; } - double inf = Py_INFINITY; + double inf = INFINITY; double x2 = nextafter(x, inf); if (isinf(x2)) { /* special case: x is the largest positive representable float */ @@ -3007,7 +3007,7 @@ math_exec(PyObject *module) if (PyModule_Add(module, "tau", PyFloat_FromDouble(Py_MATH_TAU)) < 0) { return -1; } - if (PyModule_Add(module, "inf", PyFloat_FromDouble(Py_INFINITY)) < 0) { + if (PyModule_Add(module, "inf", PyFloat_FromDouble(INFINITY)) < 0) { return -1; } if (PyModule_Add(module, "nan", PyFloat_FromDouble(fabs(Py_NAN))) < 0) { diff --git a/Objects/complexobject.c b/Objects/complexobject.c index 6247376a0e68f5..3612c2699a557d 100644 --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -139,8 +139,8 @@ _Py_c_prod(Py_complex z, Py_complex w) recalc = 1; } if (recalc) { - r.real = Py_INFINITY*(a*c - b*d); - r.imag = Py_INFINITY*(a*d + b*c); + r.real = INFINITY*(a*c - b*d); + r.imag = INFINITY*(a*d + b*c); } } @@ -229,8 +229,8 @@ _Py_c_quot(Py_complex a, Py_complex b) { const double x = copysign(isinf(a.real) ? 1.0 : 0.0, a.real); const double y = copysign(isinf(a.imag) ? 1.0 : 0.0, a.imag); - r.real = Py_INFINITY * (x*b.real + y*b.imag); - r.imag = Py_INFINITY * (y*b.real - x*b.imag); + r.real = INFINITY * (x*b.real + y*b.imag); + r.imag = INFINITY * (y*b.real - x*b.imag); } else if ((isinf(abs_breal) || isinf(abs_bimag)) && isfinite(a.real) && isfinite(a.imag)) diff --git a/Objects/floatobject.c b/Objects/floatobject.c index ef613efe4e7f44..78006783c6ec78 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -2415,7 +2415,7 @@ PyFloat_Unpack2(const char *data, int le) if (e == 0x1f) { if (f == 0) { /* Infinity */ - return sign ? -Py_INFINITY : Py_INFINITY; + return sign ? -INFINITY : INFINITY; } else { /* NaN */ diff --git a/Python/pystrtod.c b/Python/pystrtod.c index 7b74f613ed563b..e8aca939d1fb98 100644 --- a/Python/pystrtod.c +++ b/Python/pystrtod.c @@ -43,7 +43,7 @@ _Py_parse_inf_or_nan(const char *p, char **endptr) s += 3; if (case_insensitive_match(s, "inity")) s += 5; - retval = negate ? -Py_INFINITY : Py_INFINITY; + retval = negate ? -INFINITY : INFINITY; } else if (case_insensitive_match(s, "nan")) { s += 3; @@ -286,7 +286,7 @@ _PyOS_ascii_strtod(const char *nptr, char **endptr) string, -1.0 is returned and again ValueError is raised. On overflow (e.g., when trying to convert '1e500' on an IEEE 754 machine), - if overflow_exception is NULL then +-Py_INFINITY is returned, and no Python + if overflow_exception is NULL then +-INFINITY is returned, and no Python exception is raised. Otherwise, overflow_exception should point to a Python exception, this exception will be raised, -1.0 will be returned, and *endptr will point just past the end of the converted value. From f963864cb54c2e7364b2c850485c6bf25479f6f2 Mon Sep 17 00:00:00 2001 From: yihong Date: Wed, 12 Nov 2025 20:45:43 +0800 Subject: [PATCH 013/638] gh-141464: a typo in profiling sampling when can not run warning in linux (#141465) --- Lib/profiling/sampling/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/profiling/sampling/__main__.py b/Lib/profiling/sampling/__main__.py index a76ca62e2cda50..cd1425b8b9c7d3 100644 --- a/Lib/profiling/sampling/__main__.py +++ b/Lib/profiling/sampling/__main__.py @@ -15,7 +15,7 @@ """ LINUX_PERMISSION_ERROR = """ -🔒 Tachyon was unable to acess process memory. This could be because tachyon +🔒 Tachyon was unable to access process memory. This could be because tachyon has insufficient privileges (the required capability is CAP_SYS_PTRACE). Unprivileged processes cannot trace processes that they cannot send signals to or those running set-user-ID/set-group-ID programs, for security reasons. From 88aeff8eabefdc13b6fb29edb3cde618f743a034 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Wed, 12 Nov 2025 14:22:01 +0000 Subject: [PATCH 014/638] gh-87710: Update mime type for ``.ai`` (#141239) --- Doc/whatsnew/3.15.rst | 4 +++- Lib/mimetypes.py | 2 +- Lib/test/test_mimetypes.py | 1 + .../Library/2025-11-08-13-03-10.gh-issue-87710.XJeZlP.rst | 1 + 4 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-08-13-03-10.gh-issue-87710.XJeZlP.rst diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index f0fd49c9033bc1..c6089f63dee2cb 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -472,7 +472,9 @@ mimetypes * Add ``application/node`` MIME type for ``.cjs`` extension. (Contributed by John Franey in :gh:`140937`.) * Add ``application/toml``. (Contributed by Gil Forcada in :gh:`139959`.) * Rename ``application/x-texinfo`` to ``application/texinfo``. - (Contributed by Charlie Lin in :gh:`140165`) + (Contributed by Charlie Lin in :gh:`140165`.) +* Changed the MIME type for ``.ai`` files to ``application/pdf``. + (Contributed by Stan Ulbrych in :gh:`141239`.) mmap diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index d6896fc4042cb4..42477713c78418 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -497,9 +497,9 @@ def _default_mime_types(): '.oda' : 'application/oda', '.ogx' : 'application/ogg', '.pdf' : 'application/pdf', + '.ai' : 'application/pdf', '.p7c' : 'application/pkcs7-mime', '.ps' : 'application/postscript', - '.ai' : 'application/postscript', '.eps' : 'application/postscript', '.texi' : 'application/texinfo', '.texinfo': 'application/texinfo', diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py index 746984ec0ca9df..734144983591b4 100644 --- a/Lib/test/test_mimetypes.py +++ b/Lib/test/test_mimetypes.py @@ -229,6 +229,7 @@ def check_extensions(): ("application/octet-stream", ".bin"), ("application/gzip", ".gz"), ("application/ogg", ".ogx"), + ("application/pdf", ".pdf"), ("application/postscript", ".ps"), ("application/texinfo", ".texi"), ("application/toml", ".toml"), diff --git a/Misc/NEWS.d/next/Library/2025-11-08-13-03-10.gh-issue-87710.XJeZlP.rst b/Misc/NEWS.d/next/Library/2025-11-08-13-03-10.gh-issue-87710.XJeZlP.rst new file mode 100644 index 00000000000000..62073280e32b81 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-08-13-03-10.gh-issue-87710.XJeZlP.rst @@ -0,0 +1 @@ +:mod:`mimetypes`: Update mime type for ``.ai`` files to ``application/pdf``. From 2ac738d325a6934e39fecb097f43d4d4ed97a2b9 Mon Sep 17 00:00:00 2001 From: M Bussonnier Date: Wed, 12 Nov 2025 16:20:08 +0100 Subject: [PATCH 015/638] gh-132657: add regression test for `PySet_Contains` with unhashable type (#141411) --- Modules/_testlimitedcapi/set.c | 47 ++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/Modules/_testlimitedcapi/set.c b/Modules/_testlimitedcapi/set.c index 35da5fa5f008e1..34ed6b1d60b5a4 100644 --- a/Modules/_testlimitedcapi/set.c +++ b/Modules/_testlimitedcapi/set.c @@ -155,6 +155,51 @@ test_frozenset_add_in_capi(PyObject *self, PyObject *Py_UNUSED(obj)) return NULL; } +static PyObject * +test_set_contains_does_not_convert_unhashable_key(PyObject *self, PyObject *Py_UNUSED(obj)) +{ + // See https://docs.python.org/3/c-api/set.html#c.PySet_Contains + PyObject *outer_set = PySet_New(NULL); + + PyObject *needle = PySet_New(NULL); + if (needle == NULL) { + Py_DECREF(outer_set); + return NULL; + } + + PyObject *num = PyLong_FromLong(42); + if (num == NULL) { + Py_DECREF(outer_set); + Py_DECREF(needle); + return NULL; + } + + if (PySet_Add(needle, num) < 0) { + Py_DECREF(outer_set); + Py_DECREF(needle); + Py_DECREF(num); + return NULL; + } + + int result = PySet_Contains(outer_set, needle); + + Py_DECREF(num); + Py_DECREF(needle); + Py_DECREF(outer_set); + + if (result < 0) { + if (PyErr_ExceptionMatches(PyExc_TypeError)) { + PyErr_Clear(); + Py_RETURN_NONE; + } + return NULL; + } + + PyErr_SetString(PyExc_AssertionError, + "PySet_Contains should have raised TypeError for unhashable key"); + return NULL; +} + static PyMethodDef test_methods[] = { {"set_check", set_check, METH_O}, {"set_checkexact", set_checkexact, METH_O}, @@ -174,6 +219,8 @@ static PyMethodDef test_methods[] = { {"set_clear", set_clear, METH_O}, {"test_frozenset_add_in_capi", test_frozenset_add_in_capi, METH_NOARGS}, + {"test_set_contains_does_not_convert_unhashable_key", + test_set_contains_does_not_convert_unhashable_key, METH_NOARGS}, {NULL}, }; From f1330b35b8eb43904dfed0656acde80c08d63176 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Wed, 12 Nov 2025 16:37:54 +0000 Subject: [PATCH 016/638] gh-141004: Document `Py_MATH_{E, PI, TAU}` constants (#141373) --- Doc/c-api/float.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index b6020533a2b9d9..79de5daaa90d8f 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -99,6 +99,11 @@ Floating-Point Objects the C11 standard ```` header. +.. c:macro:: Py_MATH_E + + The definition (accurate for a :c:expr:`double` type) of the :data:`math.e` constant. + + .. c:macro:: Py_MATH_El High precision (long double) definition of :data:`~math.e` constant. @@ -106,6 +111,11 @@ Floating-Point Objects .. deprecated-removed:: 3.15 3.20 +.. c:macro:: Py_MATH_PI + + The definition (accurate for a :c:expr:`double` type) of the :data:`math.pi` constant. + + .. c:macro:: Py_MATH_PIl High precision (long double) definition of :data:`~math.pi` constant. @@ -113,6 +123,13 @@ Floating-Point Objects .. deprecated-removed:: 3.15 3.20 +.. c:macro:: Py_MATH_TAU + + The definition (accurate for a :c:expr:`double` type) of the :data:`math.tau` constant. + + .. versionadded:: 3.6 + + .. c:macro:: Py_RETURN_NAN Return :data:`math.nan` from a function. From 9cd5427d9619b96db20d0347a136b3d331af71ae Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 12 Nov 2025 11:38:17 -0500 Subject: [PATCH 017/638] gh-141004: Document `PyType_SUPPORTS_WEAKREFS` (GH-141408) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/c-api/type.rst | 26 ++++++++++++++++++++++++++ Doc/c-api/weakref.rst | 8 ++++++++ 2 files changed, 34 insertions(+) diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 29ffeb7c483dce..b608f815160f76 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -195,12 +195,14 @@ Type Objects before initialization) and should be paired with :c:func:`PyObject_Free` in :c:member:`~PyTypeObject.tp_free`. + .. c:function:: PyObject* PyType_GenericNew(PyTypeObject *type, PyObject *args, PyObject *kwds) Generic handler for the :c:member:`~PyTypeObject.tp_new` slot of a type object. Creates a new instance using the type's :c:member:`~PyTypeObject.tp_alloc` slot and returns the resulting object. + .. c:function:: int PyType_Ready(PyTypeObject *type) Finalize a type object. This should be called on all type objects to finish @@ -217,6 +219,7 @@ Type Objects GC protocol itself by at least implementing the :c:member:`~PyTypeObject.tp_traverse` handle. + .. c:function:: PyObject* PyType_GetName(PyTypeObject *type) Return the type's name. Equivalent to getting the type's @@ -224,6 +227,7 @@ Type Objects .. versionadded:: 3.11 + .. c:function:: PyObject* PyType_GetQualName(PyTypeObject *type) Return the type's qualified name. Equivalent to getting the @@ -239,6 +243,7 @@ Type Objects .. versionadded:: 3.13 + .. c:function:: PyObject* PyType_GetModuleName(PyTypeObject *type) Return the type's module name. Equivalent to getting the @@ -246,6 +251,7 @@ Type Objects .. versionadded:: 3.13 + .. c:function:: void* PyType_GetSlot(PyTypeObject *type, int slot) Return the function pointer stored in the given slot. If the @@ -262,6 +268,7 @@ Type Objects :c:func:`PyType_GetSlot` can now accept all types. Previously, it was limited to :ref:`heap types `. + .. c:function:: PyObject* PyType_GetModule(PyTypeObject *type) Return the module object associated with the given type when the type was @@ -281,6 +288,7 @@ Type Objects .. versionadded:: 3.9 + .. c:function:: void* PyType_GetModuleState(PyTypeObject *type) Return the state of the module object associated with the given type. @@ -295,6 +303,7 @@ Type Objects .. versionadded:: 3.9 + .. c:function:: PyObject* PyType_GetModuleByDef(PyTypeObject *type, struct PyModuleDef *def) Find the first superclass whose module was created from @@ -314,6 +323,7 @@ Type Objects .. versionadded:: 3.11 + .. c:function:: int PyType_GetBaseByToken(PyTypeObject *type, void *token, PyTypeObject **result) Find the first superclass in *type*'s :term:`method resolution order` whose @@ -332,6 +342,7 @@ Type Objects .. versionadded:: 3.14 + .. c:function:: int PyUnstable_Type_AssignVersionTag(PyTypeObject *type) Attempt to assign a version tag to the given type. @@ -342,6 +353,16 @@ Type Objects .. versionadded:: 3.12 +.. c:function:: int PyType_SUPPORTS_WEAKREFS(PyTypeObject *type) + + Return true if instances of *type* support creating weak references, false + otherwise. This function always succeeds. *type* must not be ``NULL``. + + .. seealso:: + * :ref:`weakrefobjects` + * :py:mod:`weakref` + + Creating Heap-Allocated Types ............................. @@ -390,6 +411,7 @@ The following functions and structs are used to create .. versionadded:: 3.12 + .. c:function:: PyObject* PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases) Equivalent to ``PyType_FromMetaclass(NULL, module, spec, bases)``. @@ -416,6 +438,7 @@ The following functions and structs are used to create Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. + .. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) Equivalent to ``PyType_FromMetaclass(NULL, NULL, spec, bases)``. @@ -437,6 +460,7 @@ The following functions and structs are used to create Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. + .. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec) Equivalent to ``PyType_FromMetaclass(NULL, NULL, spec, NULL)``. @@ -457,6 +481,7 @@ The following functions and structs are used to create Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. + .. c:function:: int PyType_Freeze(PyTypeObject *type) Make a type immutable: set the :c:macro:`Py_TPFLAGS_IMMUTABLETYPE` flag. @@ -628,6 +653,7 @@ The following functions and structs are used to create * :c:data:`Py_tp_token` (for clarity, prefer :c:data:`Py_TP_USE_SPEC` rather than ``NULL``) + .. c:macro:: Py_tp_token A :c:member:`~PyType_Slot.slot` that records a static memory layout ID diff --git a/Doc/c-api/weakref.rst b/Doc/c-api/weakref.rst index 39e4febd3ef0f2..db6ae0a9d4ea3d 100644 --- a/Doc/c-api/weakref.rst +++ b/Doc/c-api/weakref.rst @@ -45,6 +45,10 @@ as much as it can. weakly referenceable object, or if *callback* is not callable, ``None``, or ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. + .. seealso:: + :c:func:`PyType_SUPPORTS_WEAKREFS` for checking if *ob* is weakly + referenceable. + .. c:function:: PyObject* PyWeakref_NewProxy(PyObject *ob, PyObject *callback) @@ -57,6 +61,10 @@ as much as it can. is not a weakly referenceable object, or if *callback* is not callable, ``None``, or ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. + .. seealso:: + :c:func:`PyType_SUPPORTS_WEAKREFS` for checking if *ob* is weakly + referenceable. + .. c:function:: int PyWeakref_GetRef(PyObject *ref, PyObject **pobj) From d162c427904e232fec52d8da759caa1bfa4c01b5 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Wed, 12 Nov 2025 10:09:25 -0800 Subject: [PATCH 018/638] GH-140479: Update JIT builds to use LLVM 21 (#140973) --- .github/workflows/jit.yml | 8 ++++---- ...-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst | 1 + PCbuild/get_externals.bat | 4 ++-- Tools/jit/README.md | 20 +++++++++---------- Tools/jit/_llvm.py | 4 ++-- 5 files changed, 19 insertions(+), 18 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 69d900091a3bd1..62325250bd368e 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -68,7 +68,7 @@ jobs: - true - false llvm: - - 20 + - 21 include: - target: i686-pc-windows-msvc/msvc architecture: Win32 @@ -138,7 +138,7 @@ jobs: fail-fast: false matrix: llvm: - - 20 + - 21 steps: - uses: actions/checkout@v4 with: @@ -166,7 +166,7 @@ jobs: fail-fast: false matrix: llvm: - - 20 + - 21 steps: - uses: actions/checkout@v4 with: @@ -193,7 +193,7 @@ jobs: fail-fast: false matrix: llvm: - - 20 + - 21 steps: - uses: actions/checkout@v4 with: diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst new file mode 100644 index 00000000000000..0a615ed131127f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst @@ -0,0 +1 @@ +Update JIT compilation to use LLVM 21 at build time. diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 115203cecc8e48..9d02e2121cc623 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -82,7 +82,7 @@ if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.18 if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.15.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 -if NOT "%IncludeLLVM%"=="false" set binaries=%binaries% llvm-20.1.8.0 +if NOT "%IncludeLLVM%"=="false" set binaries=%binaries% llvm-21.1.4.0 for %%b in (%binaries%) do ( if exist "%EXTERNALS_DIR%\%%b" ( @@ -92,7 +92,7 @@ for %%b in (%binaries%) do ( git clone --depth 1 https://github.com/%ORG%/cpython-bin-deps --branch %%b "%EXTERNALS_DIR%\%%b" ) else ( echo.Fetching %%b... - if "%%b"=="llvm-20.1.8.0" ( + if "%%b"=="llvm-21.1.4.0" ( %PYTHON% -E "%PCBUILD%\get_external.py" --release --organization %ORG% --externals-dir "%EXTERNALS_DIR%" %%b ) else ( %PYTHON% -E "%PCBUILD%\get_external.py" --binary --organization %ORG% --externals-dir "%EXTERNALS_DIR%" %%b diff --git a/Tools/jit/README.md b/Tools/jit/README.md index d83b09aab59f8c..dd7deb7b256449 100644 --- a/Tools/jit/README.md +++ b/Tools/jit/README.md @@ -9,32 +9,32 @@ Python 3.11 or newer is required to build the JIT. The JIT compiler does not require end users to install any third-party dependencies, but part of it must be *built* using LLVM[^why-llvm]. You are *not* required to build the rest of CPython using LLVM, or even the same version of LLVM (in fact, this is uncommon). -LLVM version 20 is the officially supported version. You can modify if needed using the `LLVM_VERSION` env var during configure. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-19`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. +LLVM version 21 is the officially supported version. You can modify if needed using the `LLVM_VERSION` env var during configure. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-19`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. It's easy to install all of the required tools: ### Linux -Install LLVM 20 on Ubuntu/Debian: +Install LLVM 21 on Ubuntu/Debian: ```sh wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh -sudo ./llvm.sh 20 +sudo ./llvm.sh 21 ``` -Install LLVM 20 on Fedora Linux 40 or newer: +Install LLVM 21 on Fedora Linux 40 or newer: ```sh -sudo dnf install 'clang(major) = 20' 'llvm(major) = 20' +sudo dnf install 'clang(major) = 21' 'llvm(major) = 21' ``` ### macOS -Install LLVM 20 with [Homebrew](https://brew.sh): +Install LLVM 21 with [Homebrew](https://brew.sh): ```sh -brew install llvm@20 +brew install llvm@21 ``` Homebrew won't add any of the tools to your `$PATH`. That's okay; the build script knows how to find them. @@ -43,18 +43,18 @@ Homebrew won't add any of the tools to your `$PATH`. That's okay; the build scri LLVM is downloaded automatically (along with other external binary dependencies) by `PCbuild\build.bat`. -Otherwise, you can install LLVM 20 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=20), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".** +Otherwise, you can install LLVM 21 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=21), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".** Alternatively, you can use [chocolatey](https://chocolatey.org): ```sh -choco install llvm --version=20.1.8 +choco install llvm --version=21.1.0 ``` ### Dev Containers If you are working on CPython in a [Codespaces instance](https://devguide.python.org/getting-started/setup-building/#using-codespaces), there's no -need to install LLVM as the Fedora 42 base image includes LLVM 20 out of the box. +need to install LLVM as the Fedora 43 base image includes LLVM 21 out of the box. ## Building diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index f1b0ad3f5dbc43..0b9cb5192f1b75 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -11,8 +11,8 @@ import _targets -_LLVM_VERSION = "20" -_EXTERNALS_LLVM_TAG = "llvm-20.1.8.0" +_LLVM_VERSION = "21" +_EXTERNALS_LLVM_TAG = "llvm-21.1.4.0" _P = typing.ParamSpec("_P") _R = typing.TypeVar("_R") From fbcac799518e0cb29fcf5f84ed1fa001010b9073 Mon Sep 17 00:00:00 2001 From: Bob Kline Date: Wed, 12 Nov 2025 13:25:23 -0500 Subject: [PATCH 019/638] gh-141412: Use reliable target URL for urllib example (GH-141428) The endpoint used for demonstrating reading URLs is no longer stable. This change substitutes a target over which we have more control. --- Doc/tutorial/stdlib.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst index 49a3e370a4c018..342c1a00193959 100644 --- a/Doc/tutorial/stdlib.rst +++ b/Doc/tutorial/stdlib.rst @@ -183,13 +183,13 @@ protocols. Two of the simplest are :mod:`urllib.request` for retrieving data from URLs and :mod:`smtplib` for sending mail:: >>> from urllib.request import urlopen - >>> with urlopen('http://worldtimeapi.org/api/timezone/etc/UTC.txt') as response: + >>> with urlopen('https://docs.python.org/3/') as response: ... for line in response: ... line = line.decode() # Convert bytes to a str - ... if line.startswith('datetime'): + ... if 'updated' in line: ... print(line.rstrip()) # Remove trailing newline ... - datetime: 2022-01-01T01:36:47.689215+00:00 + Last updated on Nov 11, 2025 (20:11 UTC). >>> import smtplib >>> server = smtplib.SMTP('localhost') From 1f381a579cc50aa82838de84c2294b4979586bd9 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Wed, 12 Nov 2025 10:26:50 -0800 Subject: [PATCH 020/638] Add details about JIT build infrastructure and updating dependencies to `Tools/jit` (#141167) --- Tools/jit/README.md | 3 +++ Tools/jit/jit_infra.md | 28 ++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 Tools/jit/jit_infra.md diff --git a/Tools/jit/README.md b/Tools/jit/README.md index dd7deb7b256449..c70c0c47d94ad2 100644 --- a/Tools/jit/README.md +++ b/Tools/jit/README.md @@ -66,6 +66,9 @@ Otherwise, just configure and build as you normally would. Cross-compiling "just The JIT can also be enabled or disabled using the `PYTHON_JIT` environment variable, even on builds where it is enabled or disabled by default. More details about configuring CPython with the JIT and optional values for `--enable-experimental-jit` can be found [here](https://docs.python.org/dev/using/configure.html#cmdoption-enable-experimental-jit). +## Miscellaneous +If you're looking for information on how to update the JIT build dependencies, see [JIT Build Infrastructure](jit_infra.md). + [^pep-744]: [PEP 744](https://peps.python.org/pep-0744/) [^why-llvm]: Clang is specifically needed because it's the only C compiler with support for guaranteed tail calls (`musttail`), which are required by CPython's continuation-passing-style approach to JIT compilation. Since LLVM also includes other functionalities we need (namely, object file parsing and disassembly), it's convenient to only support one toolchain at this time. diff --git a/Tools/jit/jit_infra.md b/Tools/jit/jit_infra.md new file mode 100644 index 00000000000000..1a954755611d19 --- /dev/null +++ b/Tools/jit/jit_infra.md @@ -0,0 +1,28 @@ +# JIT Build Infrastructure + +This document includes details about the intricacies of the JIT build infrastructure. + +## Updating LLVM + +When we update LLVM, we need to also update the LLVM release artifact for Windows builds. This is because Windows builds automatically pull prebuilt LLVM binaries in our pipelines (e.g. notice that `.github/workflows/jit.yml` does not explicitly download LLVM or build it from source). + +To update the LLVM release artifact for Windows builds, follow these steps: +1. Go to the [LLVM releases page](https://github.com/llvm/llvm-project/releases). +1. Download x86_64 Windows artifact for the desired LLVM version (e.g. `clang+llvm-21.1.4-x86_64-pc-windows-msvc.tar.xz`). +1. Extract and repackage the tarball with the correct directory structure. For example: + ```bash + tar -xf clang+llvm-21.1.4-x86_64-pc-windows-msvc.tar.xz + mv clang+llvm-21.1.4-x86_64-pc-windows-msvc llvm-21.1.4.0 + tar -cf - llvm-21.1.4.0 | pv | xz > llvm-21.1.4.0.tar.xz + ``` + The tarball must contain a top-level directory named `llvm-{version}.0/`. +1. Go to [cpython-bin-deps](https://github.com/python/cpython-bin-deps). +1. Create a new release with the updated LLVM artifact. + - Create a new tag to match the LLVM version (e.g. `llvm-21.1.4.0`). + - Specify the release title (e.g. `LLVM 21.1.4 for x86_64 Windows`). + - Upload the asset (you can leave all other fields the same). + +### Other notes +- You must make sure that the name of the artifact matches exactly what is expected in `Tools/jit/_llvm.py` and `PCbuild/get_externals.py`. +- We don't need multiple release artifacts for each architecture because LLVM can cross-compile for different architectures on Windows; x86_64 is sufficient. +- You must have permissions to create releases in the `cpython-bin-deps` repository. If you don't have permissions, you should contact one of the organization admins. \ No newline at end of file From 35ed3e4cedc8aef3936da81a6b64e90374532b13 Mon Sep 17 00:00:00 2001 From: Mikhail Efimov Date: Wed, 12 Nov 2025 22:04:02 +0300 Subject: [PATCH 021/638] gh-140936: Fix JIT assertion crash at finalization if some generator is alive (GH-140969) --- Lib/test/test_capi/test_opt.py | 19 +++++++++++++++++++ Python/optimizer.c | 8 +++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 4e94f62d35eba2..e65556fb28f92d 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -2660,6 +2660,25 @@ def f(): f() + def test_interpreter_finalization_with_generator_alive(self): + script_helper.assert_python_ok("-c", textwrap.dedent(""" + import sys + t = tuple(range(%d)) + def simple_for(): + for x in t: + x + + def gen(): + try: + yield + except: + simple_for() + + sys.settrace(lambda *args: None) + simple_for() + g = gen() + next(g) + """ % _testinternalcapi.SPECIALIZATION_THRESHOLD)) def global_identity(x): diff --git a/Python/optimizer.c b/Python/optimizer.c index f44f8a9614b846..3b7e2dafab85bb 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -118,7 +118,13 @@ _PyOptimizer_Optimize( { _PyStackRef *stack_pointer = frame->stackpointer; PyInterpreterState *interp = _PyInterpreterState_GET(); - assert(interp->jit); + if (!interp->jit) { + // gh-140936: It is possible that interp->jit will become false during + // interpreter finalization. However, the specialized JUMP_BACKWARD_JIT + // instruction may still be present. In this case, we should + // return immediately without optimization. + return 0; + } assert(!interp->compiling); #ifndef Py_GIL_DISABLED interp->compiling = true; From 558936bec1f1e0f8346063a8cb2b2782d085178e Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Thu, 13 Nov 2025 05:41:26 +0800 Subject: [PATCH 022/638] gh-141442: Add escaping to iOS testbed arguments (#141443) Xcode concatenates the test argument array, losing quoting in the process. --- Apple/testbed/__main__.py | 3 ++- .../Tools-Demos/2025-11-12-12-54-28.gh-issue-141442.50dS3P.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2025-11-12-12-54-28.gh-issue-141442.50dS3P.rst diff --git a/Apple/testbed/__main__.py b/Apple/testbed/__main__.py index 42eb60a4c8dc02..49974cb142853c 100644 --- a/Apple/testbed/__main__.py +++ b/Apple/testbed/__main__.py @@ -2,6 +2,7 @@ import json import os import re +import shlex import shutil import subprocess import sys @@ -252,7 +253,7 @@ def update_test_plan(testbed_path, platform, args): test_plan = json.load(f) test_plan["defaultOptions"]["commandLineArgumentEntries"] = [ - {"argument": arg} for arg in args + {"argument": shlex.quote(arg)} for arg in args ] with test_plan_path.open("w", encoding="utf-8") as f: diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-11-12-12-54-28.gh-issue-141442.50dS3P.rst b/Misc/NEWS.d/next/Tools-Demos/2025-11-12-12-54-28.gh-issue-141442.50dS3P.rst new file mode 100644 index 00000000000000..073c070413f7e0 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2025-11-12-12-54-28.gh-issue-141442.50dS3P.rst @@ -0,0 +1 @@ +The iOS testbed now correctly handles test arguments that contain spaces. From dc0987080ed66c662e8e0b24cdb8c179817bd697 Mon Sep 17 00:00:00 2001 From: Michael Cho Date: Wed, 12 Nov 2025 17:16:58 -0500 Subject: [PATCH 023/638] gh-124111: Fix TCL 9 thread detection (GH-128103) --- .../Library/2025-11-12-15-42-47.gh-issue-124111.hTw4OE.rst | 2 ++ Modules/_tkinter.c | 4 ++++ 2 files changed, 6 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2025-11-12-15-42-47.gh-issue-124111.hTw4OE.rst diff --git a/Misc/NEWS.d/next/Library/2025-11-12-15-42-47.gh-issue-124111.hTw4OE.rst b/Misc/NEWS.d/next/Library/2025-11-12-15-42-47.gh-issue-124111.hTw4OE.rst new file mode 100644 index 00000000000000..8436cd2415dbd6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-12-15-42-47.gh-issue-124111.hTw4OE.rst @@ -0,0 +1,2 @@ +Updated Tcl threading configuration in :mod:`_tkinter` to assume that +threads are always available in Tcl 9 and later. diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index c0ed8977d8fd6f..8cea7b59fe730e 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -575,8 +575,12 @@ Tkapp_New(const char *screenName, const char *className, v->interp = Tcl_CreateInterp(); v->wantobjects = wantobjects; +#if TCL_MAJOR_VERSION >= 9 + v->threaded = 1; +#else v->threaded = Tcl_GetVar2Ex(v->interp, "tcl_platform", "threaded", TCL_GLOBAL_ONLY) != NULL; +#endif v->thread_id = Tcl_GetCurrentThread(); v->dispatching = 0; v->trace = NULL; From 26b7df2430cd5a9ee772bfa6ee03a73bd0b11619 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 12 Nov 2025 17:52:56 -0500 Subject: [PATCH 024/638] gh-141004: Document `PyRun_InteractiveOneObject` (GH-141405) --- Doc/c-api/veryhigh.rst | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 916c616dfee589..3b07b5fbed5959 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -100,18 +100,12 @@ the same library that the Python runtime is using. Otherwise, Python may not handle script file with LF line ending correctly. -.. c:function:: int PyRun_InteractiveOne(FILE *fp, const char *filename) - - This is a simplified interface to :c:func:`PyRun_InteractiveOneFlags` below, - leaving *flags* set to ``NULL``. - - -.. c:function:: int PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags) +.. c:function:: int PyRun_InteractiveOneObject(FILE *fp, PyObject *filename, PyCompilerFlags *flags) Read and execute a single statement from a file associated with an interactive device according to the *flags* argument. The user will be - prompted using ``sys.ps1`` and ``sys.ps2``. *filename* is decoded from the - :term:`filesystem encoding and error handler`. + prompted using ``sys.ps1`` and ``sys.ps2``. *filename* must be a Python + :class:`str` object. Returns ``0`` when the input was executed successfully, ``-1`` if there was an exception, or an error code @@ -120,6 +114,19 @@ the same library that the Python runtime is using. :file:`Python.h`, so must be included specifically if needed.) +.. c:function:: int PyRun_InteractiveOne(FILE *fp, const char *filename) + + This is a simplified interface to :c:func:`PyRun_InteractiveOneFlags` below, + leaving *flags* set to ``NULL``. + + +.. c:function:: int PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags) + + Similar to :c:func:`PyRun_InteractiveOneObject`, but *filename* is a + :c:expr:`const char*`, which is decoded from the + :term:`filesystem encoding and error handler`. + + .. c:function:: int PyRun_InteractiveLoop(FILE *fp, const char *filename) This is a simplified interface to :c:func:`PyRun_InteractiveLoopFlags` below, From 781cc68c3c814e46e6a74c3a6a32e0f9f8f7eb11 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" <68491+gpshead@users.noreply.github.com> Date: Wed, 12 Nov 2025 18:15:16 -0800 Subject: [PATCH 025/638] gh-137109: refactor warning about threads when forking (#141438) * gh-137109: refactor warning about threads when forking This splits the OS API specific functionality to get the number of threads out from the fallback Python method and warning raising code itself. This way the OS APIs can be queried before we've run `os.register_at_fork(after_in_parent=...)` registered functions which themselves may (re)start threads that would otherwise be detected. This is best effort. If the OS APIs are either unavailable or fail, the warning generating code still falls back to looking at the Python threading state after the CPython interpreter world has been restarted and the after_in_parent calls have been made. The common case for most Linux and macOS environments should work today. This also lines up with the existing TODO refactoring, we may choose to expose this API to get the number of OS threads in the `os` module in the future. * NEWS entry * avoid "function-prototype" compiler warning? --- ...-11-12-01-49-03.gh-issue-137109.D6sq2B.rst | 5 + Modules/posixmodule.c | 103 ++++++++++-------- 2 files changed, 65 insertions(+), 43 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-12-01-49-03.gh-issue-137109.D6sq2B.rst diff --git a/Misc/NEWS.d/next/Library/2025-11-12-01-49-03.gh-issue-137109.D6sq2B.rst b/Misc/NEWS.d/next/Library/2025-11-12-01-49-03.gh-issue-137109.D6sq2B.rst new file mode 100644 index 00000000000000..32f4e39f6d5f4c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-12-01-49-03.gh-issue-137109.D6sq2B.rst @@ -0,0 +1,5 @@ +The :mod:`os.fork` and related forking APIs will no longer warn in the +common case where Linux or macOS platform APIs return the number of threads +in a process and find the answer to be 1 even when a +:func:`os.register_at_fork` ``after_in_parent=`` callback (re)starts a +thread. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 6390f1fc5fe24f..fc609b2707c6c6 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -8431,53 +8431,19 @@ os_register_at_fork_impl(PyObject *module, PyObject *before, // running in the process. Best effort, silent if unable to count threads. // Constraint: Quick. Never overcounts. Never leaves an error set. // -// This should only be called from the parent process after +// This MUST only be called from the parent process after // PyOS_AfterFork_Parent(). static int -warn_about_fork_with_threads(const char* name) +warn_about_fork_with_threads( + const char* name, // Name of the API to use in the warning message. + const Py_ssize_t num_os_threads // Only trusted when >= 1. +) { // It's not safe to issue the warning while the world is stopped, because // other threads might be holding locks that we need, which would deadlock. assert(!_PyRuntime.stoptheworld.world_stopped); - // TODO: Consider making an `os` module API to return the current number - // of threads in the process. That'd presumably use this platform code but - // raise an error rather than using the inaccurate fallback. - Py_ssize_t num_python_threads = 0; -#if defined(__APPLE__) && defined(HAVE_GETPID) - mach_port_t macos_self = mach_task_self(); - mach_port_t macos_task; - if (task_for_pid(macos_self, getpid(), &macos_task) == KERN_SUCCESS) { - thread_array_t macos_threads; - mach_msg_type_number_t macos_n_threads; - if (task_threads(macos_task, &macos_threads, - &macos_n_threads) == KERN_SUCCESS) { - num_python_threads = macos_n_threads; - } - } -#elif defined(__linux__) - // Linux /proc/self/stat 20th field is the number of threads. - FILE* proc_stat = fopen("/proc/self/stat", "r"); - if (proc_stat) { - size_t n; - // Size chosen arbitrarily. ~60% more bytes than a 20th column index - // observed on the author's workstation. - char stat_line[160]; - n = fread(&stat_line, 1, 159, proc_stat); - stat_line[n] = '\0'; - fclose(proc_stat); - - char *saveptr = NULL; - char *field = strtok_r(stat_line, " ", &saveptr); - unsigned int idx; - for (idx = 19; idx && field; --idx) { - field = strtok_r(NULL, " ", &saveptr); - } - if (idx == 0 && field) { // found the 20th field - num_python_threads = atoi(field); // 0 on error - } - } -#endif + Py_ssize_t num_python_threads = num_os_threads; if (num_python_threads <= 0) { // Fall back to just the number our threading module knows about. // An incomplete view of the world, but better than nothing. @@ -8530,6 +8496,51 @@ warn_about_fork_with_threads(const char* name) } return 0; } + +// If this returns <= 0, we were unable to successfully use any OS APIs. +// Returns a positive number of threads otherwise. +static Py_ssize_t get_number_of_os_threads(void) +{ + // TODO: Consider making an `os` module API to return the current number + // of threads in the process. That'd presumably use this platform code but + // raise an error rather than using the inaccurate fallback. + Py_ssize_t num_python_threads = 0; +#if defined(__APPLE__) && defined(HAVE_GETPID) + mach_port_t macos_self = mach_task_self(); + mach_port_t macos_task; + if (task_for_pid(macos_self, getpid(), &macos_task) == KERN_SUCCESS) { + thread_array_t macos_threads; + mach_msg_type_number_t macos_n_threads; + if (task_threads(macos_task, &macos_threads, + &macos_n_threads) == KERN_SUCCESS) { + num_python_threads = macos_n_threads; + } + } +#elif defined(__linux__) + // Linux /proc/self/stat 20th field is the number of threads. + FILE* proc_stat = fopen("/proc/self/stat", "r"); + if (proc_stat) { + size_t n; + // Size chosen arbitrarily. ~60% more bytes than a 20th column index + // observed on the author's workstation. + char stat_line[160]; + n = fread(&stat_line, 1, 159, proc_stat); + stat_line[n] = '\0'; + fclose(proc_stat); + + char *saveptr = NULL; + char *field = strtok_r(stat_line, " ", &saveptr); + unsigned int idx; + for (idx = 19; idx && field; --idx) { + field = strtok_r(NULL, " ", &saveptr); + } + if (idx == 0 && field) { // found the 20th field + num_python_threads = atoi(field); // 0 on error + } + } +#endif + return num_python_threads; +} #endif // HAVE_FORK1 || HAVE_FORKPTY || HAVE_FORK #ifdef HAVE_FORK1 @@ -8564,10 +8575,12 @@ os_fork1_impl(PyObject *module) /* child: this clobbers and resets the import lock. */ PyOS_AfterFork_Child(); } else { + // Called before AfterFork_Parent in case those hooks start threads. + Py_ssize_t num_os_threads = get_number_of_os_threads(); /* parent: release the import lock. */ PyOS_AfterFork_Parent(); // After PyOS_AfterFork_Parent() starts the world to avoid deadlock. - if (warn_about_fork_with_threads("fork1") < 0) { + if (warn_about_fork_with_threads("fork1", num_os_threads) < 0) { return NULL; } } @@ -8615,10 +8628,12 @@ os_fork_impl(PyObject *module) /* child: this clobbers and resets the import lock. */ PyOS_AfterFork_Child(); } else { + // Called before AfterFork_Parent in case those hooks start threads. + Py_ssize_t num_os_threads = get_number_of_os_threads(); /* parent: release the import lock. */ PyOS_AfterFork_Parent(); // After PyOS_AfterFork_Parent() starts the world to avoid deadlock. - if (warn_about_fork_with_threads("fork") < 0) + if (warn_about_fork_with_threads("fork", num_os_threads) < 0) return NULL; } if (pid == -1) { @@ -9476,6 +9491,8 @@ os_forkpty_impl(PyObject *module) /* child: this clobbers and resets the import lock. */ PyOS_AfterFork_Child(); } else { + // Called before AfterFork_Parent in case those hooks start threads. + Py_ssize_t num_os_threads = get_number_of_os_threads(); /* parent: release the import lock. */ PyOS_AfterFork_Parent(); /* set O_CLOEXEC on master_fd */ @@ -9485,7 +9502,7 @@ os_forkpty_impl(PyObject *module) } // After PyOS_AfterFork_Parent() starts the world to avoid deadlock. - if (warn_about_fork_with_threads("forkpty") < 0) + if (warn_about_fork_with_threads("forkpty", num_os_threads) < 0) return NULL; } if (pid == -1) { From 63548b36998e7f7cd5c7c28b53b348a93f836737 Mon Sep 17 00:00:00 2001 From: Shamil Date: Thu, 13 Nov 2025 14:01:31 +0300 Subject: [PATCH 026/638] gh-140260: fix data race in `_struct` module initialization with subinterpreters (#140909) --- Lib/test/test_struct.py | 17 ++++ ...-11-02-15-28-33.gh-issue-140260.JNzlGz.rst | 2 + Modules/_struct.c | 91 ++++++++++--------- Tools/c-analyzer/cpython/ignored.tsv | 1 + 4 files changed, 70 insertions(+), 41 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-15-28-33.gh-issue-140260.JNzlGz.rst diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py index 75c76a36ee92f5..cceecdd526c006 100644 --- a/Lib/test/test_struct.py +++ b/Lib/test/test_struct.py @@ -800,6 +800,23 @@ def test_c_complex_round_trip(self): round_trip = struct.unpack(f, struct.pack(f, z))[0] self.assertComplexesAreIdentical(z, round_trip) + @unittest.skipIf( + support.is_android or support.is_apple_mobile, + "Subinterpreters are not supported on Android and iOS" + ) + def test_endian_table_init_subinterpreters(self): + # Verify that the _struct extension module can be initialized + # concurrently in subinterpreters (gh-140260). + try: + from concurrent.futures import InterpreterPoolExecutor + except ImportError: + raise unittest.SkipTest("InterpreterPoolExecutor not available") + + code = "import struct" + with InterpreterPoolExecutor(max_workers=5) as executor: + results = executor.map(exec, [code] * 5) + self.assertListEqual(list(results), [None] * 5) + class UnpackIteratorTest(unittest.TestCase): """ diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-15-28-33.gh-issue-140260.JNzlGz.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-15-28-33.gh-issue-140260.JNzlGz.rst new file mode 100644 index 00000000000000..96bf9b51e4862c --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-15-28-33.gh-issue-140260.JNzlGz.rst @@ -0,0 +1,2 @@ +Fix :mod:`struct` data race in endian table initialization with +subinterpreters. Patch by Shamil Abdulaev. diff --git a/Modules/_struct.c b/Modules/_struct.c index f09252e82c3915..2acb3df3a30395 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -9,6 +9,7 @@ #include "Python.h" #include "pycore_bytesobject.h" // _PyBytesWriter +#include "pycore_lock.h" // _PyOnceFlag_CallOnce() #include "pycore_long.h" // _PyLong_AsByteArray() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() @@ -1505,6 +1506,53 @@ static formatdef lilendian_table[] = { {0} }; +/* Ensure endian table optimization happens exactly once across all interpreters */ +static _PyOnceFlag endian_tables_init_once = {0}; + +static int +init_endian_tables(void *Py_UNUSED(arg)) +{ + const formatdef *native = native_table; + formatdef *other, *ptr; +#if PY_LITTLE_ENDIAN + other = lilendian_table; +#else + other = bigendian_table; +#endif + /* Scan through the native table, find a matching + entry in the endian table and swap in the + native implementations whenever possible + (64-bit platforms may not have "standard" sizes) */ + while (native->format != '\0' && other->format != '\0') { + ptr = other; + while (ptr->format != '\0') { + if (ptr->format == native->format) { + /* Match faster when formats are + listed in the same order */ + if (ptr == other) + other++; + /* Only use the trick if the + size matches */ + if (ptr->size != native->size) + break; + /* Skip float and double, could be + "unknown" float format */ + if (ptr->format == 'd' || ptr->format == 'f') + break; + /* Skip _Bool, semantics are different for standard size */ + if (ptr->format == '?') + break; + ptr->pack = native->pack; + ptr->unpack = native->unpack; + break; + } + ptr++; + } + native++; + } + return 0; +} + static const formatdef * whichtable(const char **pfmt) @@ -2710,47 +2758,8 @@ _structmodule_exec(PyObject *m) return -1; } - /* Check endian and swap in faster functions */ - { - const formatdef *native = native_table; - formatdef *other, *ptr; -#if PY_LITTLE_ENDIAN - other = lilendian_table; -#else - other = bigendian_table; -#endif - /* Scan through the native table, find a matching - entry in the endian table and swap in the - native implementations whenever possible - (64-bit platforms may not have "standard" sizes) */ - while (native->format != '\0' && other->format != '\0') { - ptr = other; - while (ptr->format != '\0') { - if (ptr->format == native->format) { - /* Match faster when formats are - listed in the same order */ - if (ptr == other) - other++; - /* Only use the trick if the - size matches */ - if (ptr->size != native->size) - break; - /* Skip float and double, could be - "unknown" float format */ - if (ptr->format == 'd' || ptr->format == 'f') - break; - /* Skip _Bool, semantics are different for standard size */ - if (ptr->format == '?') - break; - ptr->pack = native->pack; - ptr->unpack = native->unpack; - break; - } - ptr++; - } - native++; - } - } + /* init cannot fail */ + (void)_PyOnceFlag_CallOnce(&endian_tables_init_once, init_endian_tables, NULL); /* Add some symbolic constants to the module */ state->StructError = PyErr_NewException("struct.error", NULL, NULL); diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 11a3cd794ff4d7..4621ad250f4633 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -24,6 +24,7 @@ Modules/posixmodule.c os_dup2_impl dup3_works - ## guards around resource init Python/thread_pthread.h PyThread__init_thread lib_initialized - +Modules/_struct.c - endian_tables_init_once - ##----------------------- ## other values (not Python-specific) From d8e6bdc0d083f4e76ac49574544555ad91257592 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 13 Nov 2025 13:21:32 +0200 Subject: [PATCH 027/638] gh-135801: Add the module parameter to compile() etc (GH-139652) Many functions related to compiling or parsing Python code, such as compile(), ast.parse(), symtable.symtable(), and importlib.abc.InspectLoader.source_to_code() now allow to pass the module name used when filtering syntax warnings. --- Doc/library/ast.rst | 7 ++- Doc/library/functions.rst | 11 +++- Doc/library/importlib.rst | 10 +++- Doc/library/symtable.rst | 8 ++- Doc/whatsnew/3.15.rst | 7 +++ Include/internal/pycore_compile.h | 9 ++- Include/internal/pycore_parser.h | 3 +- Include/internal/pycore_pyerrors.h | 3 +- Include/internal/pycore_pythonrun.h | 6 ++ Include/internal/pycore_symtable.h | 3 +- Lib/ast.py | 5 +- Lib/importlib/_bootstrap_external.py | 9 +-- Lib/importlib/abc.py | 11 ++-- Lib/modulefinder.py | 2 +- Lib/profiling/sampling/_sync_coordinator.py | 2 +- Lib/profiling/tracing/__init__.py | 2 +- Lib/runpy.py | 6 +- Lib/symtable.py | 4 +- Lib/test/test_ast/test_ast.py | 10 ++++ Lib/test/test_builtin.py | 3 +- Lib/test/test_cmd_line_script.py | 23 ++++++-- Lib/test/test_compile.py | 10 ++++ Lib/test/test_import/__init__.py | 15 +---- Lib/test/test_runpy.py | 43 ++++++++++++++ Lib/test/test_symtable.py | 10 ++++ Lib/test/test_zipimport_support.py | 23 ++++++++ Lib/zipimport.py | 6 +- ...-10-06-14-19-47.gh-issue-135801.OhxEZS.rst | 6 ++ Modules/clinic/symtablemodule.c.h | 58 ++++++++++++++++--- Modules/symtablemodule.c | 19 +++++- Parser/lexer/state.c | 2 + Parser/lexer/state.h | 1 + Parser/peg_api.c | 6 +- Parser/pegen.c | 8 ++- Parser/pegen.h | 2 +- Parser/string_parser.c | 2 +- Parser/tokenizer/helpers.c | 4 +- Programs/_freeze_module.py | 2 +- Programs/freeze_test_frozenmain.py | 2 +- Python/ast_preprocess.c | 8 ++- Python/bltinmodule.c | 25 ++++++-- Python/clinic/bltinmodule.c.h | 26 ++++++--- Python/compile.c | 30 ++++++---- Python/errors.c | 7 ++- Python/pythonrun.c | 38 ++++++++++-- Python/symtable.c | 4 +- .../peg_extension/peg_extension.c | 4 +- 47 files changed, 390 insertions(+), 115 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-14-19-47.gh-issue-135801.OhxEZS.rst diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 494621672171f2..0ea3c3c59a660d 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -2205,10 +2205,10 @@ Async and await Apart from the node classes, the :mod:`ast` module defines these utility functions and classes for traversing abstract syntax trees: -.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None, optimize=-1) +.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None, optimize=-1, module=None) Parse the source into an AST node. Equivalent to ``compile(source, - filename, mode, flags=FLAGS_VALUE, optimize=optimize)``, + filename, mode, flags=FLAGS_VALUE, optimize=optimize, module=module)``, where ``FLAGS_VALUE`` is ``ast.PyCF_ONLY_AST`` if ``optimize <= 0`` and ``ast.PyCF_OPTIMIZED_AST`` otherwise. @@ -2261,6 +2261,9 @@ and classes for traversing abstract syntax trees: The minimum supported version for ``feature_version`` is now ``(3, 7)``. The ``optimize`` argument was added. + .. versionadded:: next + Added the *module* parameter. + .. function:: unparse(ast_obj) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index e98793975556ef..3257daf89d327b 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -292,7 +292,9 @@ are always available. They are listed here in alphabetical order. :func:`property`. -.. function:: compile(source, filename, mode, flags=0, dont_inherit=False, optimize=-1) +.. function:: compile(source, filename, mode, flags=0, \ + dont_inherit=False, optimize=-1, \ + *, module=None) Compile the *source* into a code or AST object. Code objects can be executed by :func:`exec` or :func:`eval`. *source* can either be a normal string, a @@ -334,6 +336,10 @@ are always available. They are listed here in alphabetical order. ``__debug__`` is true), ``1`` (asserts are removed, ``__debug__`` is false) or ``2`` (docstrings are removed too). + The optional argument *module* specifies the module name. + It is needed to unambiguous :ref:`filter ` syntax warnings + by module name. + This function raises :exc:`SyntaxError` if the compiled source is invalid, and :exc:`ValueError` if the source contains null bytes. @@ -371,6 +377,9 @@ are always available. They are listed here in alphabetical order. ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` can now be passed in flags to enable support for top-level ``await``, ``async for``, and ``async with``. + .. versionadded:: next + Added the *module* parameter. + .. class:: complex(number=0, /) complex(string, /) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 602a7100a12350..03ba23b6216cbf 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -459,7 +459,7 @@ ABC hierarchy:: .. versionchanged:: 3.4 Raises :exc:`ImportError` instead of :exc:`NotImplementedError`. - .. staticmethod:: source_to_code(data, path='') + .. staticmethod:: source_to_code(data, path='', fullname=None) Create a code object from Python source. @@ -471,11 +471,19 @@ ABC hierarchy:: With the subsequent code object one can execute it in a module by running ``exec(code, module.__dict__)``. + The optional argument *fullname* specifies the module name. + It is needed to unambiguous :ref:`filter ` syntax + warnings by module name. + .. versionadded:: 3.4 .. versionchanged:: 3.5 Made the method static. + .. versionadded:: next + Added the *fullname* parameter. + + .. method:: exec_module(module) Implementation of :meth:`Loader.exec_module`. diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst index 54e19af4bd69a6..c0d9e79197de7c 100644 --- a/Doc/library/symtable.rst +++ b/Doc/library/symtable.rst @@ -21,11 +21,17 @@ tables. Generating Symbol Tables ------------------------ -.. function:: symtable(code, filename, compile_type) +.. function:: symtable(code, filename, compile_type, *, module=None) Return the toplevel :class:`SymbolTable` for the Python source *code*. *filename* is the name of the file containing the code. *compile_type* is like the *mode* argument to :func:`compile`. + The optional argument *module* specifies the module name. + It is needed to unambiguous :ref:`filter ` syntax warnings + by module name. + + .. versionadded:: next + Added the *module* parameter. Examining Symbol Tables diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index c6089f63dee2cb..3cb766978a7217 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -307,6 +307,13 @@ Other language changes not only integers or floats, although this does not improve precision. (Contributed by Serhiy Storchaka in :gh:`67795`.) +* Many functions related to compiling or parsing Python code, such as + :func:`compile`, :func:`ast.parse`, :func:`symtable.symtable`, + and :func:`importlib.abc.InspectLoader.source_to_code`, now allow to pass + the module name. It is needed to unambiguous :ref:`filter ` + syntax warnings by module name. + (Contributed by Serhiy Storchaka in :gh:`135801`.) + New modules =========== diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index 1c60834fa2058c..527141b54d0dca 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -32,7 +32,8 @@ PyAPI_FUNC(PyCodeObject*) _PyAST_Compile( PyObject *filename, PyCompilerFlags *flags, int optimize, - struct _arena *arena); + struct _arena *arena, + PyObject *module); /* AST preprocessing */ extern int _PyCompile_AstPreprocess( @@ -41,7 +42,8 @@ extern int _PyCompile_AstPreprocess( PyCompilerFlags *flags, int optimize, struct _arena *arena, - int syntax_check_only); + int syntax_check_only, + PyObject *module); extern int _PyAST_Preprocess( struct _mod *, @@ -50,7 +52,8 @@ extern int _PyAST_Preprocess( int optimize, int ff_features, int syntax_check_only, - int enable_warnings); + int enable_warnings, + PyObject *module); typedef struct { diff --git a/Include/internal/pycore_parser.h b/Include/internal/pycore_parser.h index 2885dee63dcf94..2c46f59ab7da9f 100644 --- a/Include/internal/pycore_parser.h +++ b/Include/internal/pycore_parser.h @@ -48,7 +48,8 @@ extern struct _mod* _PyParser_ASTFromString( PyObject* filename, int mode, PyCompilerFlags *flags, - PyArena *arena); + PyArena *arena, + PyObject *module); extern struct _mod* _PyParser_ASTFromFile( FILE *fp, diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index 2c2048f7e1272a..f80808fcc8c4d7 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -123,7 +123,8 @@ extern void _PyErr_SetNone(PyThreadState *tstate, PyObject *exception); extern PyObject* _PyErr_NoMemory(PyThreadState *tstate); extern int _PyErr_EmitSyntaxWarning(PyObject *msg, PyObject *filename, int lineno, int col_offset, - int end_lineno, int end_col_offset); + int end_lineno, int end_col_offset, + PyObject *module); extern void _PyErr_RaiseSyntaxError(PyObject *msg, PyObject *filename, int lineno, int col_offset, int end_lineno, int end_col_offset); diff --git a/Include/internal/pycore_pythonrun.h b/Include/internal/pycore_pythonrun.h index c2832098ddb3e7..f954f1b63ef67c 100644 --- a/Include/internal/pycore_pythonrun.h +++ b/Include/internal/pycore_pythonrun.h @@ -33,6 +33,12 @@ extern const char* _Py_SourceAsString( PyCompilerFlags *cf, PyObject **cmd_copy); +extern PyObject * _Py_CompileStringObjectWithModule( + const char *str, + PyObject *filename, int start, + PyCompilerFlags *flags, int optimize, + PyObject *module); + /* Stack size, in "pointers". This must be large enough, so * no two calls to check recursion depth are more than this far diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index 98099b4a497b01..9dbfa913219afa 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -188,7 +188,8 @@ extern struct symtable* _Py_SymtableStringObjectFlags( const char *str, PyObject *filename, int start, - PyCompilerFlags *flags); + PyCompilerFlags *flags, + PyObject *module); int _PyFuture_FromAST( struct _mod * mod, diff --git a/Lib/ast.py b/Lib/ast.py index 983ac1710d0205..d9743ba7ab40b1 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -24,7 +24,7 @@ def parse(source, filename='', mode='exec', *, - type_comments=False, feature_version=None, optimize=-1): + type_comments=False, feature_version=None, optimize=-1, module=None): """ Parse the source into an AST node. Equivalent to compile(source, filename, mode, PyCF_ONLY_AST). @@ -44,7 +44,8 @@ def parse(source, filename='', mode='exec', *, feature_version = minor # Else it should be an int giving the minor version for 3.x. return compile(source, filename, mode, flags, - _feature_version=feature_version, optimize=optimize) + _feature_version=feature_version, optimize=optimize, + module=module) def literal_eval(node_or_string): diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 035ae0fcae14e8..4ab0e79ea6efeb 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -819,13 +819,14 @@ def get_source(self, fullname): name=fullname) from exc return decode_source(source_bytes) - def source_to_code(self, data, path, *, _optimize=-1): + def source_to_code(self, data, path, fullname=None, *, _optimize=-1): """Return the code object compiled from source. The 'data' argument can be any object type that compile() supports. """ return _bootstrap._call_with_frames_removed(compile, data, path, 'exec', - dont_inherit=True, optimize=_optimize) + dont_inherit=True, optimize=_optimize, + module=fullname) def get_code(self, fullname): """Concrete implementation of InspectLoader.get_code. @@ -894,7 +895,7 @@ def get_code(self, fullname): source_path=source_path) if source_bytes is None: source_bytes = self.get_data(source_path) - code_object = self.source_to_code(source_bytes, source_path) + code_object = self.source_to_code(source_bytes, source_path, fullname) _bootstrap._verbose_message('code object from {}', source_path) if (not sys.dont_write_bytecode and bytecode_path is not None and source_mtime is not None): @@ -1186,7 +1187,7 @@ def get_source(self, fullname): return '' def get_code(self, fullname): - return compile('', '', 'exec', dont_inherit=True) + return compile('', '', 'exec', dont_inherit=True, module=fullname) def create_module(self, spec): """Use default semantics for module creation.""" diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index 1e47495f65fa02..5c13432b5bda8c 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -108,7 +108,7 @@ def get_code(self, fullname): source = self.get_source(fullname) if source is None: return None - return self.source_to_code(source) + return self.source_to_code(source, '', fullname) @abc.abstractmethod def get_source(self, fullname): @@ -120,12 +120,12 @@ def get_source(self, fullname): raise ImportError @staticmethod - def source_to_code(data, path=''): + def source_to_code(data, path='', fullname=None): """Compile 'data' into a code object. The 'data' argument can be anything that compile() can handle. The'path' argument should be where the data was retrieved (when applicable).""" - return compile(data, path, 'exec', dont_inherit=True) + return compile(data, path, 'exec', dont_inherit=True, module=fullname) exec_module = _bootstrap_external._LoaderBasics.exec_module load_module = _bootstrap_external._LoaderBasics.load_module @@ -163,9 +163,8 @@ def get_code(self, fullname): try: path = self.get_filename(fullname) except ImportError: - return self.source_to_code(source) - else: - return self.source_to_code(source, path) + path = '' + return self.source_to_code(source, path, fullname) _register( ExecutionLoader, diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py index ac478ee7f51722..b115d99ab30ff1 100644 --- a/Lib/modulefinder.py +++ b/Lib/modulefinder.py @@ -334,7 +334,7 @@ def load_module(self, fqname, fp, pathname, file_info): self.msgout(2, "load_module ->", m) return m if type == _PY_SOURCE: - co = compile(fp.read(), pathname, 'exec') + co = compile(fp.read(), pathname, 'exec', module=fqname) elif type == _PY_COMPILED: try: data = fp.read() diff --git a/Lib/profiling/sampling/_sync_coordinator.py b/Lib/profiling/sampling/_sync_coordinator.py index 8716e654104791..adb040e89cc7b1 100644 --- a/Lib/profiling/sampling/_sync_coordinator.py +++ b/Lib/profiling/sampling/_sync_coordinator.py @@ -182,7 +182,7 @@ def _execute_script(script_path: str, script_args: List[str], cwd: str) -> None: try: # Compile and execute the script - code = compile(source_code, script_path, 'exec') + code = compile(source_code, script_path, 'exec', module='__main__') exec(code, {'__name__': '__main__', '__file__': script_path}) except SyntaxError as e: raise TargetError(f"Syntax error in script {script_path}: {e}") from e diff --git a/Lib/profiling/tracing/__init__.py b/Lib/profiling/tracing/__init__.py index 2dc7ea92c8ca4d..a6b8edf721611f 100644 --- a/Lib/profiling/tracing/__init__.py +++ b/Lib/profiling/tracing/__init__.py @@ -185,7 +185,7 @@ def main(): progname = args[0] sys.path.insert(0, os.path.dirname(progname)) with io.open_code(progname) as fp: - code = compile(fp.read(), progname, 'exec') + code = compile(fp.read(), progname, 'exec', module='__main__') spec = importlib.machinery.ModuleSpec(name='__main__', loader=None, origin=progname) module = importlib.util.module_from_spec(spec) diff --git a/Lib/runpy.py b/Lib/runpy.py index ef54d3282eee06..f072498f6cb405 100644 --- a/Lib/runpy.py +++ b/Lib/runpy.py @@ -247,7 +247,7 @@ def _get_main_module_details(error=ImportError): sys.modules[main_name] = saved_main -def _get_code_from_file(fname): +def _get_code_from_file(fname, module): # Check for a compiled file first from pkgutil import read_code code_path = os.path.abspath(fname) @@ -256,7 +256,7 @@ def _get_code_from_file(fname): if code is None: # That didn't work, so try it as normal source code with io.open_code(code_path) as f: - code = compile(f.read(), fname, 'exec') + code = compile(f.read(), fname, 'exec', module=module) return code def run_path(path_name, init_globals=None, run_name=None): @@ -283,7 +283,7 @@ def run_path(path_name, init_globals=None, run_name=None): if isinstance(importer, type(None)): # Not a valid sys.path entry, so run the code directly # execfile() doesn't help as we want to allow compiled files - code = _get_code_from_file(path_name) + code = _get_code_from_file(path_name, run_name) return _run_module_code(code, init_globals, run_name, pkg_name=pkg_name, script_name=path_name) else: diff --git a/Lib/symtable.py b/Lib/symtable.py index 77475c3ffd9224..4c832e68f94cbd 100644 --- a/Lib/symtable.py +++ b/Lib/symtable.py @@ -17,13 +17,13 @@ __all__ = ["symtable", "SymbolTableType", "SymbolTable", "Class", "Function", "Symbol"] -def symtable(code, filename, compile_type): +def symtable(code, filename, compile_type, *, module=None): """ Return the toplevel *SymbolTable* for the source code. *filename* is the name of the file with the code and *compile_type* is the *compile()* mode argument. """ - top = _symtable.symtable(code, filename, compile_type) + top = _symtable.symtable(code, filename, compile_type, module=module) return _newSymbolTable(top, filename) class SymbolTableFactory: diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 551de5851daace..fb4a441ca64772 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -1083,6 +1083,16 @@ def test_filter_syntax_warnings_by_module(self): self.assertEqual(wm.filename, '') self.assertIs(wm.category, SyntaxWarning) + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'package\.module\z') + warnings.filterwarnings('error', module=r'') + ast.parse(source, filename, module='package.module') + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10]) + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + class CopyTests(unittest.TestCase): """Test copying and pickling AST nodes.""" diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index fba46af6617640..ce60a5d095dd52 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -1103,7 +1103,8 @@ def test_exec_filter_syntax_warnings_by_module(self): with warnings.catch_warnings(record=True) as wlog: warnings.simplefilter('error') - warnings.filterwarnings('always', module=r'\z') + warnings.filterwarnings('always', module=r'package.module\z') + warnings.filterwarnings('error', module=r'') exec(source, {'__name__': 'package.module', '__file__': filename}) self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) for wm in wlog: diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index f8115cc8300df7..cc1a625a5097d8 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -814,15 +814,26 @@ def test_filter_syntax_warnings_by_module(self): filename = support.findfile('test_import/data/syntax_warnings.py') rc, out, err = assert_python_ok( '-Werror', - '-Walways:::test.test_import.data.syntax_warnings', + '-Walways:::__main__', + '-Werror:::test.test_import.data.syntax_warnings', + '-Werror:::syntax_warnings', filename) self.assertEqual(err.count(b': SyntaxWarning: '), 6) - rc, out, err = assert_python_ok( - '-Werror', - '-Walways:::syntax_warnings', - filename) - self.assertEqual(err.count(b': SyntaxWarning: '), 6) + def test_zipfile_run_filter_syntax_warnings_by_module(self): + filename = support.findfile('test_import/data/syntax_warnings.py') + with open(filename, 'rb') as f: + source = f.read() + with os_helper.temp_dir() as script_dir: + zip_name, _ = make_zip_pkg( + script_dir, 'test_zip', 'test_pkg', '__main__', source) + rc, out, err = assert_python_ok( + '-Werror', + '-Walways:::__main__', + '-Werror:::test_pkg.__main__', + os.path.join(zip_name, 'test_pkg') + ) + self.assertEqual(err.count(b': SyntaxWarning: '), 12) def tearDownModule(): diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 9c2364491fe08d..30f21875b22ab3 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1759,6 +1759,16 @@ def test_filter_syntax_warnings_by_module(self): self.assertEqual(wm.filename, filename) self.assertIs(wm.category, SyntaxWarning) + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'package\.module\z') + warnings.filterwarnings('error', module=module_re) + compile(source, filename, 'exec', module='package.module') + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + @support.subTests('src', [ textwrap.dedent(""" def f(): diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index e87d8b7e7bbb1f..fe669bb04df02a 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -1259,20 +1259,7 @@ def test_filter_syntax_warnings_by_module(self): warnings.catch_warnings(record=True) as wlog): warnings.simplefilter('error') warnings.filterwarnings('always', module=module_re) - import test.test_import.data.syntax_warnings - self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) - filename = test.test_import.data.syntax_warnings.__file__ - for wm in wlog: - self.assertEqual(wm.filename, filename) - self.assertIs(wm.category, SyntaxWarning) - - module_re = r'syntax_warnings\z' - unload('test.test_import.data.syntax_warnings') - with (os_helper.temp_dir() as tmpdir, - temporary_pycache_prefix(tmpdir), - warnings.catch_warnings(record=True) as wlog): - warnings.simplefilter('error') - warnings.filterwarnings('always', module=module_re) + warnings.filterwarnings('error', module='syntax_warnings') import test.test_import.data.syntax_warnings self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) filename = test.test_import.data.syntax_warnings.__file__ diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py index a2a07c04f58ef2..cc76b72b9639eb 100644 --- a/Lib/test/test_runpy.py +++ b/Lib/test/test_runpy.py @@ -20,9 +20,11 @@ requires_subprocess, verbose, ) +from test import support from test.support.import_helper import forget, make_legacy_pyc, unload from test.support.os_helper import create_empty_file, temp_dir, FakePath from test.support.script_helper import make_script, make_zip_script +from test.test_importlib.util import temporary_pycache_prefix import runpy @@ -763,6 +765,47 @@ def test_encoding(self): result = run_path(filename) self.assertEqual(result['s'], "non-ASCII: h\xe9") + def test_run_module_filter_syntax_warnings_by_module(self): + module_re = r'test\.test_import\.data\.syntax_warnings\z' + with (temp_dir() as tmpdir, + temporary_pycache_prefix(tmpdir), + warnings.catch_warnings(record=True) as wlog): + warnings.simplefilter('error') + warnings.filterwarnings('always', module=module_re) + warnings.filterwarnings('error', module='syntax_warnings') + ns = run_module('test.test_import.data.syntax_warnings') + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + filename = ns['__file__'] + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + + def test_run_path_filter_syntax_warnings_by_module(self): + filename = support.findfile('test_import/data/syntax_warnings.py') + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'\z') + warnings.filterwarnings('error', module='test') + warnings.filterwarnings('error', module='syntax_warnings') + warnings.filterwarnings('error', + module=r'test\.test_import\.data\.syntax_warnings') + run_path(filename) + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'package\.script\z') + warnings.filterwarnings('error', module='') + warnings.filterwarnings('error', module='test') + warnings.filterwarnings('error', module='syntax_warnings') + warnings.filterwarnings('error', + module=r'test\.test_import\.data\.syntax_warnings') + run_path(filename, run_name='package.script') + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10, 13, 14, 21]) + @force_not_colorized_test_class class TestExit(unittest.TestCase): diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index ef2c00e04b820c..094ab8f573e7ba 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -601,6 +601,16 @@ def test_filter_syntax_warnings_by_module(self): self.assertEqual(wm.filename, filename) self.assertIs(wm.category, SyntaxWarning) + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'package\.module\z') + warnings.filterwarnings('error', module=module_re) + symtable.symtable(source, filename, 'exec', module='package.module') + self.assertEqual(sorted(wm.lineno for wm in wlog), [4, 7, 10]) + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + class ComprehensionTests(unittest.TestCase): def get_identifiers_recursive(self, st, res): diff --git a/Lib/test/test_zipimport_support.py b/Lib/test/test_zipimport_support.py index ae8a8c99762313..2b28f46149b4ff 100644 --- a/Lib/test/test_zipimport_support.py +++ b/Lib/test/test_zipimport_support.py @@ -13,9 +13,12 @@ import inspect import linecache import unittest +import warnings +from test import support from test.support import os_helper from test.support.script_helper import (spawn_python, kill_python, assert_python_ok, make_script, make_zip_script) +from test.support import import_helper verbose = test.support.verbose @@ -236,6 +239,26 @@ def f(): # bdb/pdb applies normcase to its filename before displaying self.assertIn(os.path.normcase(run_name.encode('utf-8')), data) + def test_import_filter_syntax_warnings_by_module(self): + filename = support.findfile('test_import/data/syntax_warnings.py') + with (os_helper.temp_dir() as tmpdir, + import_helper.DirsOnSysPath()): + zip_name, _ = make_zip_script(tmpdir, "test_zip", + filename, 'test_pkg/test_mod.py') + sys.path.insert(0, zip_name) + import_helper.unload('test_pkg.test_mod') + with warnings.catch_warnings(record=True) as wlog: + warnings.simplefilter('error') + warnings.filterwarnings('always', module=r'test_pkg\.test_mod\z') + warnings.filterwarnings('error', module='test_mod') + import test_pkg.test_mod + self.assertEqual(sorted(wm.lineno for wm in wlog), + sorted([4, 7, 10, 13, 14, 21]*2)) + filename = test_pkg.test_mod.__file__ + for wm in wlog: + self.assertEqual(wm.filename, filename) + self.assertIs(wm.category, SyntaxWarning) + def tearDownModule(): test.support.reap_children() diff --git a/Lib/zipimport.py b/Lib/zipimport.py index 340a7e07112504..19279d1c2bea36 100644 --- a/Lib/zipimport.py +++ b/Lib/zipimport.py @@ -742,9 +742,9 @@ def _normalize_line_endings(source): # Given a string buffer containing Python source code, compile it # and return a code object. -def _compile_source(pathname, source): +def _compile_source(pathname, source, module): source = _normalize_line_endings(source) - return compile(source, pathname, 'exec', dont_inherit=True) + return compile(source, pathname, 'exec', dont_inherit=True, module=module) # Convert the date/time values found in the Zip archive to a value # that's compatible with the time stamp stored in .pyc files. @@ -815,7 +815,7 @@ def _get_module_code(self, fullname): except ImportError as exc: import_error = exc else: - code = _compile_source(modpath, data) + code = _compile_source(modpath, data, fullname) if code is None: # bad magic number or non-matching mtime # in byte code, try next diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-14-19-47.gh-issue-135801.OhxEZS.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-14-19-47.gh-issue-135801.OhxEZS.rst new file mode 100644 index 00000000000000..96226a7c525e80 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-14-19-47.gh-issue-135801.OhxEZS.rst @@ -0,0 +1,6 @@ +Many functions related to compiling or parsing Python code, such as +:func:`compile`, :func:`ast.parse`, :func:`symtable.symtable`, and +:func:`importlib.abc.InspectLoader.source_to_code` now allow to specify +the module name. +It is needed to unambiguous :ref:`filter ` syntax warnings +by module name. diff --git a/Modules/clinic/symtablemodule.c.h b/Modules/clinic/symtablemodule.c.h index bd55d77c5409e9..65352593f94802 100644 --- a/Modules/clinic/symtablemodule.c.h +++ b/Modules/clinic/symtablemodule.c.h @@ -2,30 +2,67 @@ preserve [clinic start generated code]*/ -#include "pycore_modsupport.h" // _PyArg_CheckPositional() +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_symtable_symtable__doc__, -"symtable($module, source, filename, startstr, /)\n" +"symtable($module, source, filename, startstr, /, *, module=None)\n" "--\n" "\n" "Return symbol and scope dictionaries used internally by compiler."); #define _SYMTABLE_SYMTABLE_METHODDEF \ - {"symtable", _PyCFunction_CAST(_symtable_symtable), METH_FASTCALL, _symtable_symtable__doc__}, + {"symtable", _PyCFunction_CAST(_symtable_symtable), METH_FASTCALL|METH_KEYWORDS, _symtable_symtable__doc__}, static PyObject * _symtable_symtable_impl(PyObject *module, PyObject *source, - PyObject *filename, const char *startstr); + PyObject *filename, const char *startstr, + PyObject *modname); static PyObject * -_symtable_symtable(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +_symtable_symtable(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(module), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "", "", "module", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "symtable", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; PyObject *source; PyObject *filename = NULL; const char *startstr; + PyObject *modname = Py_None; - if (!_PyArg_CheckPositional("symtable", nargs, 3, 3)) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 3, /*maxpos*/ 3, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { goto exit; } source = args[0]; @@ -45,7 +82,12 @@ _symtable_symtable(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyErr_SetString(PyExc_ValueError, "embedded null character"); goto exit; } - return_value = _symtable_symtable_impl(module, source, filename, startstr); + if (!noptargs) { + goto skip_optional_kwonly; + } + modname = args[3]; +skip_optional_kwonly: + return_value = _symtable_symtable_impl(module, source, filename, startstr, modname); exit: /* Cleanup for filename */ @@ -53,4 +95,4 @@ _symtable_symtable(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } -/*[clinic end generated code: output=7a8545d9a1efe837 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0137be60c487c841 input=a9049054013a1b77]*/ diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c index d353f406831ecd..a24927a9db64db 100644 --- a/Modules/symtablemodule.c +++ b/Modules/symtablemodule.c @@ -16,14 +16,17 @@ _symtable.symtable filename: unicode_fs_decoded startstr: str / + * + module as modname: object = None Return symbol and scope dictionaries used internally by compiler. [clinic start generated code]*/ static PyObject * _symtable_symtable_impl(PyObject *module, PyObject *source, - PyObject *filename, const char *startstr) -/*[clinic end generated code: output=59eb0d5fc7285ac4 input=436ffff90d02e4f6]*/ + PyObject *filename, const char *startstr, + PyObject *modname) +/*[clinic end generated code: output=235ec5a87a9ce178 input=fbf9adaa33c7070d]*/ { struct symtable *st; PyObject *t; @@ -50,7 +53,17 @@ _symtable_symtable_impl(PyObject *module, PyObject *source, Py_XDECREF(source_copy); return NULL; } - st = _Py_SymtableStringObjectFlags(str, filename, start, &cf); + if (modname == Py_None) { + modname = NULL; + } + else if (!PyUnicode_Check(modname)) { + PyErr_Format(PyExc_TypeError, + "symtable() argument 'module' must be str or None, not %T", + modname); + Py_XDECREF(source_copy); + return NULL; + } + st = _Py_SymtableStringObjectFlags(str, filename, start, &cf, modname); Py_XDECREF(source_copy); if (st == NULL) { return NULL; diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c index 2de9004fe084f2..3663dc3eb7f9f6 100644 --- a/Parser/lexer/state.c +++ b/Parser/lexer/state.c @@ -43,6 +43,7 @@ _PyTokenizer_tok_new(void) tok->encoding = NULL; tok->cont_line = 0; tok->filename = NULL; + tok->module = NULL; tok->decoding_readline = NULL; tok->decoding_buffer = NULL; tok->readline = NULL; @@ -91,6 +92,7 @@ _PyTokenizer_Free(struct tok_state *tok) Py_XDECREF(tok->decoding_buffer); Py_XDECREF(tok->readline); Py_XDECREF(tok->filename); + Py_XDECREF(tok->module); if ((tok->readline != NULL || tok->fp != NULL ) && tok->buf != NULL) { PyMem_Free(tok->buf); } diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h index 877127125a7652..9cd196a114c7cb 100644 --- a/Parser/lexer/state.h +++ b/Parser/lexer/state.h @@ -102,6 +102,7 @@ struct tok_state { int parenlinenostack[MAXLEVEL]; int parencolstack[MAXLEVEL]; PyObject *filename; + PyObject *module; /* Stuff for checking on different tab sizes */ int altindstack[MAXINDENT]; /* Stack of alternate indents */ /* Stuff for PEP 0263 */ diff --git a/Parser/peg_api.c b/Parser/peg_api.c index d4acc3e4935d10..e30ca0453bd3e1 100644 --- a/Parser/peg_api.c +++ b/Parser/peg_api.c @@ -4,13 +4,15 @@ mod_ty _PyParser_ASTFromString(const char *str, PyObject* filename, int mode, - PyCompilerFlags *flags, PyArena *arena) + PyCompilerFlags *flags, PyArena *arena, + PyObject *module) { if (PySys_Audit("compile", "yO", str, filename) < 0) { return NULL; } - mod_ty result = _PyPegen_run_parser_from_string(str, mode, filename, flags, arena); + mod_ty result = _PyPegen_run_parser_from_string(str, mode, filename, flags, + arena, module); return result; } diff --git a/Parser/pegen.c b/Parser/pegen.c index 70493031656028..a38e973b3f64c6 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -1010,6 +1010,11 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena // From here on we need to clean up even if there's an error mod_ty result = NULL; + tok->module = PyUnicode_FromString("__main__"); + if (tok->module == NULL) { + goto error; + } + int parser_flags = compute_parser_flags(flags); Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, PY_MINOR_VERSION, errcode, NULL, arena); @@ -1036,7 +1041,7 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena mod_ty _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filename_ob, - PyCompilerFlags *flags, PyArena *arena) + PyCompilerFlags *flags, PyArena *arena, PyObject *module) { int exec_input = start_rule == Py_file_input; @@ -1054,6 +1059,7 @@ _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filen } // This transfers the ownership to the tokenizer tok->filename = Py_NewRef(filename_ob); + tok->module = Py_XNewRef(module); // We need to clear up from here on mod_ty result = NULL; diff --git a/Parser/pegen.h b/Parser/pegen.h index 6b49b3537a04b2..b8f887608b104e 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -378,7 +378,7 @@ mod_ty _PyPegen_run_parser_from_file_pointer(FILE *, int, PyObject *, const char const char *, const char *, PyCompilerFlags *, int *, PyObject **, PyArena *); void *_PyPegen_run_parser(Parser *); -mod_ty _PyPegen_run_parser_from_string(const char *, int, PyObject *, PyCompilerFlags *, PyArena *); +mod_ty _PyPegen_run_parser_from_string(const char *, int, PyObject *, PyCompilerFlags *, PyArena *, PyObject *); asdl_stmt_seq *_PyPegen_interactive_exit(Parser *); // Generated function in parse.c - function definition in python.gram diff --git a/Parser/string_parser.c b/Parser/string_parser.c index ebe68989d1af58..b164dfbc81a933 100644 --- a/Parser/string_parser.c +++ b/Parser/string_parser.c @@ -88,7 +88,7 @@ warn_invalid_escape_sequence(Parser *p, const char* buffer, const char *first_in } if (PyErr_WarnExplicitObject(category, msg, p->tok->filename, - lineno, NULL, NULL) < 0) { + lineno, p->tok->module, NULL) < 0) { if (PyErr_ExceptionMatches(category)) { /* Replace the Syntax/DeprecationWarning exception with a SyntaxError to get a more accurate error report */ diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c index e5e2eed2d34aee..a03531a744136d 100644 --- a/Parser/tokenizer/helpers.c +++ b/Parser/tokenizer/helpers.c @@ -127,7 +127,7 @@ _PyTokenizer_warn_invalid_escape_sequence(struct tok_state *tok, int first_inval } if (PyErr_WarnExplicitObject(PyExc_SyntaxWarning, msg, tok->filename, - tok->lineno, NULL, NULL) < 0) { + tok->lineno, tok->module, NULL) < 0) { Py_DECREF(msg); if (PyErr_ExceptionMatches(PyExc_SyntaxWarning)) { @@ -166,7 +166,7 @@ _PyTokenizer_parser_warn(struct tok_state *tok, PyObject *category, const char * } if (PyErr_WarnExplicitObject(category, errmsg, tok->filename, - tok->lineno, NULL, NULL) < 0) { + tok->lineno, tok->module, NULL) < 0) { if (PyErr_ExceptionMatches(category)) { /* Replace the DeprecationWarning exception with a SyntaxError to get a more accurate error report */ diff --git a/Programs/_freeze_module.py b/Programs/_freeze_module.py index ba638eef6c4cd6..62274e4aa9ce11 100644 --- a/Programs/_freeze_module.py +++ b/Programs/_freeze_module.py @@ -23,7 +23,7 @@ def read_text(inpath: str) -> bytes: def compile_and_marshal(name: str, text: bytes) -> bytes: filename = f"" # exec == Py_file_input - code = compile(text, filename, "exec", optimize=0, dont_inherit=True) + code = compile(text, filename, "exec", optimize=0, dont_inherit=True, module=name) return marshal.dumps(code) diff --git a/Programs/freeze_test_frozenmain.py b/Programs/freeze_test_frozenmain.py index 848fc31b3d6f44..1a986bbac2afc7 100644 --- a/Programs/freeze_test_frozenmain.py +++ b/Programs/freeze_test_frozenmain.py @@ -24,7 +24,7 @@ def dump(fp, filename, name): with tokenize.open(filename) as source_fp: source = source_fp.read() - code = compile(source, code_filename, 'exec') + code = compile(source, code_filename, 'exec', module=name) data = marshal.dumps(code) writecode(fp, name, data) diff --git a/Python/ast_preprocess.c b/Python/ast_preprocess.c index fe6fd9479d1531..d45435257cc8ac 100644 --- a/Python/ast_preprocess.c +++ b/Python/ast_preprocess.c @@ -16,6 +16,7 @@ typedef struct { typedef struct { PyObject *filename; + PyObject *module; int optimize; int ff_features; int syntax_check_only; @@ -71,7 +72,8 @@ control_flow_in_finally_warning(const char *kw, stmt_ty n, _PyASTPreprocessState } int ret = _PyErr_EmitSyntaxWarning(msg, state->filename, n->lineno, n->col_offset + 1, n->end_lineno, - n->end_col_offset + 1); + n->end_col_offset + 1, + state->module); Py_DECREF(msg); return ret < 0 ? 0 : 1; } @@ -969,11 +971,13 @@ astfold_type_param(type_param_ty node_, PyArena *ctx_, _PyASTPreprocessState *st int _PyAST_Preprocess(mod_ty mod, PyArena *arena, PyObject *filename, int optimize, - int ff_features, int syntax_check_only, int enable_warnings) + int ff_features, int syntax_check_only, int enable_warnings, + PyObject *module) { _PyASTPreprocessState state; memset(&state, 0, sizeof(_PyASTPreprocessState)); state.filename = filename; + state.module = module; state.optimize = optimize; state.ff_features = ff_features; state.syntax_check_only = syntax_check_only; diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index f6fadd936bb8ff..c2d780ac9b9270 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -751,6 +751,7 @@ compile as builtin_compile dont_inherit: bool = False optimize: int = -1 * + module as modname: object = None _feature_version as feature_version: int = -1 Compile source into a code object that can be executed by exec() or eval(). @@ -770,8 +771,8 @@ in addition to any features explicitly specified. static PyObject * builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, const char *mode, int flags, int dont_inherit, - int optimize, int feature_version) -/*[clinic end generated code: output=b0c09c84f116d3d7 input=8f0069edbdac381b]*/ + int optimize, PyObject *modname, int feature_version) +/*[clinic end generated code: output=9a0dce1945917a86 input=ddeae1e0253459dc]*/ { PyObject *source_copy; const char *str; @@ -800,6 +801,15 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, "compile(): invalid optimize value"); goto error; } + if (modname == Py_None) { + modname = NULL; + } + else if (!PyUnicode_Check(modname)) { + PyErr_Format(PyExc_TypeError, + "compile() argument 'module' must be str or None, not %T", + modname); + goto error; + } if (!dont_inherit) { PyEval_MergeCompilerFlags(&cf); @@ -845,8 +855,9 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, goto error; } int syntax_check_only = ((flags & PyCF_OPTIMIZED_AST) == PyCF_ONLY_AST); /* unoptiomized AST */ - if (_PyCompile_AstPreprocess(mod, filename, &cf, optimize, - arena, syntax_check_only) < 0) { + if (_PyCompile_AstPreprocess(mod, filename, &cf, optimize, arena, + syntax_check_only, modname) < 0) + { _PyArena_Free(arena); goto error; } @@ -859,7 +870,7 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, goto error; } result = (PyObject*)_PyAST_Compile(mod, filename, - &cf, optimize, arena); + &cf, optimize, arena, modname); } _PyArena_Free(arena); goto finally; @@ -877,7 +888,9 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, tstate->suppress_co_const_immortalization++; #endif - result = Py_CompileStringObject(str, filename, start[compile_mode], &cf, optimize); + result = _Py_CompileStringObjectWithModule(str, filename, + start[compile_mode], &cf, + optimize, modname); #ifdef Py_GIL_DISABLED tstate->suppress_co_const_immortalization--; diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index adb82f45c25b5d..f08e5847abe32a 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -238,7 +238,8 @@ PyDoc_STRVAR(builtin_chr__doc__, PyDoc_STRVAR(builtin_compile__doc__, "compile($module, /, source, filename, mode, flags=0,\n" -" dont_inherit=False, optimize=-1, *, _feature_version=-1)\n" +" dont_inherit=False, optimize=-1, *, module=None,\n" +" _feature_version=-1)\n" "--\n" "\n" "Compile source into a code object that can be executed by exec() or eval().\n" @@ -260,7 +261,7 @@ PyDoc_STRVAR(builtin_compile__doc__, static PyObject * builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, const char *mode, int flags, int dont_inherit, - int optimize, int feature_version); + int optimize, PyObject *modname, int feature_version); static PyObject * builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -268,7 +269,7 @@ builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 7 + #define NUM_KEYWORDS 8 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -277,7 +278,7 @@ builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(source), &_Py_ID(filename), &_Py_ID(mode), &_Py_ID(flags), &_Py_ID(dont_inherit), &_Py_ID(optimize), &_Py_ID(_feature_version), }, + .ob_item = { &_Py_ID(source), &_Py_ID(filename), &_Py_ID(mode), &_Py_ID(flags), &_Py_ID(dont_inherit), &_Py_ID(optimize), &_Py_ID(module), &_Py_ID(_feature_version), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -286,14 +287,14 @@ builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"source", "filename", "mode", "flags", "dont_inherit", "optimize", "_feature_version", NULL}; + static const char * const _keywords[] = {"source", "filename", "mode", "flags", "dont_inherit", "optimize", "module", "_feature_version", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "compile", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[7]; + PyObject *argsbuf[8]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; PyObject *source; PyObject *filename = NULL; @@ -301,6 +302,7 @@ builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj int flags = 0; int dont_inherit = 0; int optimize = -1; + PyObject *modname = Py_None; int feature_version = -1; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, @@ -359,12 +361,18 @@ builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj if (!noptargs) { goto skip_optional_kwonly; } - feature_version = PyLong_AsInt(args[6]); + if (args[6]) { + modname = args[6]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + feature_version = PyLong_AsInt(args[7]); if (feature_version == -1 && PyErr_Occurred()) { goto exit; } skip_optional_kwonly: - return_value = builtin_compile_impl(module, source, filename, mode, flags, dont_inherit, optimize, feature_version); + return_value = builtin_compile_impl(module, source, filename, mode, flags, dont_inherit, optimize, modname, feature_version); exit: /* Cleanup for filename */ @@ -1277,4 +1285,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=7eada753dc2e046f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=06500bcc9a341e68 input=a9049054013a1b77]*/ diff --git a/Python/compile.c b/Python/compile.c index e2f1c7e8eb5bce..6951c98500dfec 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -104,11 +104,13 @@ typedef struct _PyCompiler { * (including instructions for nested code objects) */ int c_disable_warning; + PyObject *c_module; } compiler; static int compiler_setup(compiler *c, mod_ty mod, PyObject *filename, - PyCompilerFlags *flags, int optimize, PyArena *arena) + PyCompilerFlags *flags, int optimize, PyArena *arena, + PyObject *module) { PyCompilerFlags local_flags = _PyCompilerFlags_INIT; @@ -126,6 +128,7 @@ compiler_setup(compiler *c, mod_ty mod, PyObject *filename, if (!_PyFuture_FromAST(mod, filename, &c->c_future)) { return ERROR; } + c->c_module = Py_XNewRef(module); if (!flags) { flags = &local_flags; } @@ -136,7 +139,9 @@ compiler_setup(compiler *c, mod_ty mod, PyObject *filename, c->c_optimize = (optimize == -1) ? _Py_GetConfig()->optimization_level : optimize; c->c_save_nested_seqs = false; - if (!_PyAST_Preprocess(mod, arena, filename, c->c_optimize, merged, 0, 1)) { + if (!_PyAST_Preprocess(mod, arena, filename, c->c_optimize, merged, + 0, 1, module)) + { return ERROR; } c->c_st = _PySymtable_Build(mod, filename, &c->c_future); @@ -156,6 +161,7 @@ compiler_free(compiler *c) _PySymtable_Free(c->c_st); } Py_XDECREF(c->c_filename); + Py_XDECREF(c->c_module); Py_XDECREF(c->c_const_cache); Py_XDECREF(c->c_stack); PyMem_Free(c); @@ -163,13 +169,13 @@ compiler_free(compiler *c) static compiler* new_compiler(mod_ty mod, PyObject *filename, PyCompilerFlags *pflags, - int optimize, PyArena *arena) + int optimize, PyArena *arena, PyObject *module) { compiler *c = PyMem_Calloc(1, sizeof(compiler)); if (c == NULL) { return NULL; } - if (compiler_setup(c, mod, filename, pflags, optimize, arena) < 0) { + if (compiler_setup(c, mod, filename, pflags, optimize, arena, module) < 0) { compiler_free(c); return NULL; } @@ -1221,7 +1227,8 @@ _PyCompile_Warn(compiler *c, location loc, const char *format, ...) return ERROR; } int ret = _PyErr_EmitSyntaxWarning(msg, c->c_filename, loc.lineno, loc.col_offset + 1, - loc.end_lineno, loc.end_col_offset + 1); + loc.end_lineno, loc.end_col_offset + 1, + c->c_module); Py_DECREF(msg); return ret; } @@ -1476,10 +1483,10 @@ _PyCompile_OptimizeAndAssemble(compiler *c, int addNone) PyCodeObject * _PyAST_Compile(mod_ty mod, PyObject *filename, PyCompilerFlags *pflags, - int optimize, PyArena *arena) + int optimize, PyArena *arena, PyObject *module) { assert(!PyErr_Occurred()); - compiler *c = new_compiler(mod, filename, pflags, optimize, arena); + compiler *c = new_compiler(mod, filename, pflags, optimize, arena, module); if (c == NULL) { return NULL; } @@ -1492,7 +1499,8 @@ _PyAST_Compile(mod_ty mod, PyObject *filename, PyCompilerFlags *pflags, int _PyCompile_AstPreprocess(mod_ty mod, PyObject *filename, PyCompilerFlags *cf, - int optimize, PyArena *arena, int no_const_folding) + int optimize, PyArena *arena, int no_const_folding, + PyObject *module) { _PyFutureFeatures future; if (!_PyFuture_FromAST(mod, filename, &future)) { @@ -1502,7 +1510,9 @@ _PyCompile_AstPreprocess(mod_ty mod, PyObject *filename, PyCompilerFlags *cf, if (optimize == -1) { optimize = _Py_GetConfig()->optimization_level; } - if (!_PyAST_Preprocess(mod, arena, filename, optimize, flags, no_const_folding, 0)) { + if (!_PyAST_Preprocess(mod, arena, filename, optimize, flags, + no_const_folding, 0, module)) + { return -1; } return 0; @@ -1627,7 +1637,7 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, return NULL; } - compiler *c = new_compiler(mod, filename, pflags, optimize, arena); + compiler *c = new_compiler(mod, filename, pflags, optimize, arena, NULL); if (c == NULL) { _PyArena_Free(arena); return NULL; diff --git a/Python/errors.c b/Python/errors.c index 9fe95cec0ab794..5c6ac48371a0ff 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -1960,10 +1960,11 @@ _PyErr_RaiseSyntaxError(PyObject *msg, PyObject *filename, int lineno, int col_o */ int _PyErr_EmitSyntaxWarning(PyObject *msg, PyObject *filename, int lineno, int col_offset, - int end_lineno, int end_col_offset) + int end_lineno, int end_col_offset, + PyObject *module) { - if (PyErr_WarnExplicitObject(PyExc_SyntaxWarning, msg, - filename, lineno, NULL, NULL) < 0) + if (PyErr_WarnExplicitObject(PyExc_SyntaxWarning, msg, filename, lineno, + module, NULL) < 0) { if (PyErr_ExceptionMatches(PyExc_SyntaxWarning)) { /* Replace the SyntaxWarning exception with a SyntaxError diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 45211e1b075042..49ce0a97d4742f 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1252,12 +1252,19 @@ _PyRun_StringFlagsWithName(const char *str, PyObject* name, int start, } else { name = &_Py_STR(anon_string); } + PyObject *module = NULL; + if (globals && PyDict_GetItemStringRef(globals, "__name__", &module) < 0) { + goto done; + } - mod = _PyParser_ASTFromString(str, name, start, flags, arena); + mod = _PyParser_ASTFromString(str, name, start, flags, arena, module); + Py_XDECREF(module); - if (mod != NULL) { + if (mod != NULL) { ret = run_mod(mod, name, globals, locals, flags, arena, source, generate_new_source); } + +done: Py_XDECREF(source); _PyArena_Free(arena); return ret; @@ -1407,8 +1414,17 @@ run_mod(mod_ty mod, PyObject *filename, PyObject *globals, PyObject *locals, return NULL; } } + PyObject *module = NULL; + if (globals && PyDict_GetItemStringRef(globals, "__name__", &module) < 0) { + if (interactive_src) { + Py_DECREF(interactive_filename); + } + return NULL; + } - PyCodeObject *co = _PyAST_Compile(mod, interactive_filename, flags, -1, arena); + PyCodeObject *co = _PyAST_Compile(mod, interactive_filename, flags, -1, + arena, module); + Py_XDECREF(module); if (co == NULL) { if (interactive_src) { Py_DECREF(interactive_filename); @@ -1507,6 +1523,14 @@ run_pyc_file(FILE *fp, PyObject *globals, PyObject *locals, PyObject * Py_CompileStringObject(const char *str, PyObject *filename, int start, PyCompilerFlags *flags, int optimize) +{ + return _Py_CompileStringObjectWithModule(str, filename, start, + flags, optimize, NULL); +} + +PyObject * +_Py_CompileStringObjectWithModule(const char *str, PyObject *filename, int start, + PyCompilerFlags *flags, int optimize, PyObject *module) { PyCodeObject *co; mod_ty mod; @@ -1514,14 +1538,16 @@ Py_CompileStringObject(const char *str, PyObject *filename, int start, if (arena == NULL) return NULL; - mod = _PyParser_ASTFromString(str, filename, start, flags, arena); + mod = _PyParser_ASTFromString(str, filename, start, flags, arena, module); if (mod == NULL) { _PyArena_Free(arena); return NULL; } if (flags && (flags->cf_flags & PyCF_ONLY_AST)) { int syntax_check_only = ((flags->cf_flags & PyCF_OPTIMIZED_AST) == PyCF_ONLY_AST); /* unoptiomized AST */ - if (_PyCompile_AstPreprocess(mod, filename, flags, optimize, arena, syntax_check_only) < 0) { + if (_PyCompile_AstPreprocess(mod, filename, flags, optimize, arena, + syntax_check_only, module) < 0) + { _PyArena_Free(arena); return NULL; } @@ -1529,7 +1555,7 @@ Py_CompileStringObject(const char *str, PyObject *filename, int start, _PyArena_Free(arena); return result; } - co = _PyAST_Compile(mod, filename, flags, optimize, arena); + co = _PyAST_Compile(mod, filename, flags, optimize, arena, module); _PyArena_Free(arena); return (PyObject *)co; } diff --git a/Python/symtable.c b/Python/symtable.c index bcd7365f8e1f14..29cf9190a4e95b 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -3137,7 +3137,7 @@ symtable_raise_if_not_coroutine(struct symtable *st, const char *msg, _Py_Source struct symtable * _Py_SymtableStringObjectFlags(const char *str, PyObject *filename, - int start, PyCompilerFlags *flags) + int start, PyCompilerFlags *flags, PyObject *module) { struct symtable *st; mod_ty mod; @@ -3147,7 +3147,7 @@ _Py_SymtableStringObjectFlags(const char *str, PyObject *filename, if (arena == NULL) return NULL; - mod = _PyParser_ASTFromString(str, filename, start, flags, arena); + mod = _PyParser_ASTFromString(str, filename, start, flags, arena, module); if (mod == NULL) { _PyArena_Free(arena); return NULL; diff --git a/Tools/peg_generator/peg_extension/peg_extension.c b/Tools/peg_generator/peg_extension/peg_extension.c index 1587d53d59472e..2fec5b0512940f 100644 --- a/Tools/peg_generator/peg_extension/peg_extension.c +++ b/Tools/peg_generator/peg_extension/peg_extension.c @@ -8,7 +8,7 @@ _build_return_object(mod_ty module, int mode, PyObject *filename_ob, PyArena *ar PyObject *result = NULL; if (mode == 2) { - result = (PyObject *)_PyAST_Compile(module, filename_ob, NULL, -1, arena); + result = (PyObject *)_PyAST_Compile(module, filename_ob, NULL, -1, arena, NULL); } else if (mode == 1) { result = PyAST_mod2obj(module); } else { @@ -93,7 +93,7 @@ parse_string(PyObject *self, PyObject *args, PyObject *kwds) PyCompilerFlags flags = _PyCompilerFlags_INIT; mod_ty res = _PyPegen_run_parser_from_string(the_string, Py_file_input, filename_ob, - &flags, arena); + &flags, arena, NULL); if (res == NULL) { goto error; } From 2fbd39666663cb5ca1c0e3021ce2e7bc72331020 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 13 Nov 2025 13:37:01 +0200 Subject: [PATCH 028/638] gh-140601: Refactor ElementTree.iterparse() tests (GH-141499) Split existing tests on smaller methods and move them to separate class. Rename variable "content" to "it". Use BytesIO instead of StringIO. Add few more tests. --- Lib/test/test_xml_etree.py | 430 ++++++++++++++++++++----------------- 1 file changed, 228 insertions(+), 202 deletions(-) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index f65baa0cfae2ad..25c084c8b9c9eb 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -574,208 +574,6 @@ def test_parseliteral(self): self.assertEqual(len(ids), 1) self.assertEqual(ids["body"].tag, 'body') - def test_iterparse(self): - # Test iterparse interface. - - iterparse = ET.iterparse - - context = iterparse(SIMPLE_XMLFILE) - self.assertIsNone(context.root) - action, elem = next(context) - self.assertIsNone(context.root) - self.assertEqual((action, elem.tag), ('end', 'element')) - self.assertEqual([(action, elem.tag) for action, elem in context], [ - ('end', 'element'), - ('end', 'empty-element'), - ('end', 'root'), - ]) - self.assertEqual(context.root.tag, 'root') - - context = iterparse(SIMPLE_NS_XMLFILE) - self.assertEqual([(action, elem.tag) for action, elem in context], [ - ('end', '{namespace}element'), - ('end', '{namespace}element'), - ('end', '{namespace}empty-element'), - ('end', '{namespace}root'), - ]) - - with open(SIMPLE_XMLFILE, 'rb') as source: - context = iterparse(source) - action, elem = next(context) - self.assertEqual((action, elem.tag), ('end', 'element')) - self.assertEqual([(action, elem.tag) for action, elem in context], [ - ('end', 'element'), - ('end', 'empty-element'), - ('end', 'root'), - ]) - self.assertEqual(context.root.tag, 'root') - - events = () - context = iterparse(SIMPLE_XMLFILE, events) - self.assertEqual([(action, elem.tag) for action, elem in context], []) - - events = () - context = iterparse(SIMPLE_XMLFILE, events=events) - self.assertEqual([(action, elem.tag) for action, elem in context], []) - - events = ("start", "end") - context = iterparse(SIMPLE_XMLFILE, events) - self.assertEqual([(action, elem.tag) for action, elem in context], [ - ('start', 'root'), - ('start', 'element'), - ('end', 'element'), - ('start', 'element'), - ('end', 'element'), - ('start', 'empty-element'), - ('end', 'empty-element'), - ('end', 'root'), - ]) - - events = ("start", "end", "start-ns", "end-ns") - context = iterparse(SIMPLE_NS_XMLFILE, events) - self.assertEqual([(action, elem.tag) if action in ("start", "end") - else (action, elem) - for action, elem in context], [ - ('start-ns', ('', 'namespace')), - ('start', '{namespace}root'), - ('start', '{namespace}element'), - ('end', '{namespace}element'), - ('start', '{namespace}element'), - ('end', '{namespace}element'), - ('start', '{namespace}empty-element'), - ('end', '{namespace}empty-element'), - ('end', '{namespace}root'), - ('end-ns', None), - ]) - - events = ('start-ns', 'end-ns') - context = iterparse(io.StringIO(r""), events) - res = [action for action, elem in context] - self.assertEqual(res, ['start-ns', 'end-ns']) - - events = ("start", "end", "bogus") - with open(SIMPLE_XMLFILE, "rb") as f: - with self.assertRaises(ValueError) as cm: - iterparse(f, events) - self.assertFalse(f.closed) - self.assertEqual(str(cm.exception), "unknown event 'bogus'") - - with warnings_helper.check_no_resource_warning(self): - with self.assertRaises(ValueError) as cm: - iterparse(SIMPLE_XMLFILE, events) - self.assertEqual(str(cm.exception), "unknown event 'bogus'") - del cm - - source = io.BytesIO( - b"\n" - b"text\n") - events = ("start-ns",) - context = iterparse(source, events) - self.assertEqual([(action, elem) for action, elem in context], [ - ('start-ns', ('', 'http://\xe9ffbot.org/ns')), - ('start-ns', ('cl\xe9', 'http://effbot.org/ns')), - ]) - - source = io.StringIO("junk") - it = iterparse(source) - action, elem = next(it) - self.assertEqual((action, elem.tag), ('end', 'document')) - with self.assertRaises(ET.ParseError) as cm: - next(it) - self.assertEqual(str(cm.exception), - 'junk after document element: line 1, column 12') - - self.addCleanup(os_helper.unlink, TESTFN) - with open(TESTFN, "wb") as f: - f.write(b"junk") - it = iterparse(TESTFN) - action, elem = next(it) - self.assertEqual((action, elem.tag), ('end', 'document')) - with warnings_helper.check_no_resource_warning(self): - with self.assertRaises(ET.ParseError) as cm: - next(it) - self.assertEqual(str(cm.exception), - 'junk after document element: line 1, column 12') - del cm, it - - # Not exhausting the iterator still closes the resource (bpo-43292) - with warnings_helper.check_no_resource_warning(self): - it = iterparse(SIMPLE_XMLFILE) - del it - - with warnings_helper.check_no_resource_warning(self): - it = iterparse(SIMPLE_XMLFILE) - it.close() - del it - - with warnings_helper.check_no_resource_warning(self): - it = iterparse(SIMPLE_XMLFILE) - action, elem = next(it) - self.assertEqual((action, elem.tag), ('end', 'element')) - del it, elem - - with warnings_helper.check_no_resource_warning(self): - it = iterparse(SIMPLE_XMLFILE) - action, elem = next(it) - it.close() - self.assertEqual((action, elem.tag), ('end', 'element')) - del it, elem - - with self.assertRaises(FileNotFoundError): - iterparse("nonexistent") - - def test_iterparse_close(self): - iterparse = ET.iterparse - - it = iterparse(SIMPLE_XMLFILE) - it.close() - with self.assertRaises(StopIteration): - next(it) - it.close() # idempotent - - with open(SIMPLE_XMLFILE, 'rb') as source: - it = iterparse(source) - it.close() - self.assertFalse(source.closed) - with self.assertRaises(StopIteration): - next(it) - it.close() # idempotent - - it = iterparse(SIMPLE_XMLFILE) - action, elem = next(it) - self.assertEqual((action, elem.tag), ('end', 'element')) - it.close() - with self.assertRaises(StopIteration): - next(it) - it.close() # idempotent - - with open(SIMPLE_XMLFILE, 'rb') as source: - it = iterparse(source) - action, elem = next(it) - self.assertEqual((action, elem.tag), ('end', 'element')) - it.close() - self.assertFalse(source.closed) - with self.assertRaises(StopIteration): - next(it) - it.close() # idempotent - - it = iterparse(SIMPLE_XMLFILE) - list(it) - it.close() - with self.assertRaises(StopIteration): - next(it) - it.close() # idempotent - - with open(SIMPLE_XMLFILE, 'rb') as source: - it = iterparse(source) - list(it) - it.close() - self.assertFalse(source.closed) - with self.assertRaises(StopIteration): - next(it) - it.close() # idempotent - def test_writefile(self): elem = ET.Element("tag") elem.text = "text" @@ -1499,6 +1297,234 @@ def test_attlist_default(self): {'{http://www.w3.org/XML/1998/namespace}lang': 'eng'}) +class IterparseTest(unittest.TestCase): + # Test iterparse interface. + + def test_basic(self): + iterparse = ET.iterparse + + it = iterparse(SIMPLE_XMLFILE) + self.assertIsNone(it.root) + action, elem = next(it) + self.assertIsNone(it.root) + self.assertEqual((action, elem.tag), ('end', 'element')) + self.assertEqual([(action, elem.tag) for action, elem in it], [ + ('end', 'element'), + ('end', 'empty-element'), + ('end', 'root'), + ]) + self.assertEqual(it.root.tag, 'root') + it.close() + + it = iterparse(SIMPLE_NS_XMLFILE) + self.assertEqual([(action, elem.tag) for action, elem in it], [ + ('end', '{namespace}element'), + ('end', '{namespace}element'), + ('end', '{namespace}empty-element'), + ('end', '{namespace}root'), + ]) + it.close() + + def test_external_file(self): + with open(SIMPLE_XMLFILE, 'rb') as source: + it = ET.iterparse(source) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + self.assertEqual([(action, elem.tag) for action, elem in it], [ + ('end', 'element'), + ('end', 'empty-element'), + ('end', 'root'), + ]) + self.assertEqual(it.root.tag, 'root') + + def test_events(self): + iterparse = ET.iterparse + + events = () + it = iterparse(SIMPLE_XMLFILE, events) + self.assertEqual([(action, elem.tag) for action, elem in it], []) + it.close() + + events = () + it = iterparse(SIMPLE_XMLFILE, events=events) + self.assertEqual([(action, elem.tag) for action, elem in it], []) + it.close() + + events = ("start", "end") + it = iterparse(SIMPLE_XMLFILE, events) + self.assertEqual([(action, elem.tag) for action, elem in it], [ + ('start', 'root'), + ('start', 'element'), + ('end', 'element'), + ('start', 'element'), + ('end', 'element'), + ('start', 'empty-element'), + ('end', 'empty-element'), + ('end', 'root'), + ]) + it.close() + + def test_namespace_events(self): + iterparse = ET.iterparse + + events = ("start", "end", "start-ns", "end-ns") + it = iterparse(SIMPLE_NS_XMLFILE, events) + self.assertEqual([(action, elem.tag) if action in ("start", "end") + else (action, elem) + for action, elem in it], [ + ('start-ns', ('', 'namespace')), + ('start', '{namespace}root'), + ('start', '{namespace}element'), + ('end', '{namespace}element'), + ('start', '{namespace}element'), + ('end', '{namespace}element'), + ('start', '{namespace}empty-element'), + ('end', '{namespace}empty-element'), + ('end', '{namespace}root'), + ('end-ns', None), + ]) + it.close() + + events = ('start-ns', 'end-ns') + it = iterparse(io.BytesIO(br""), events) + res = [action for action, elem in it] + self.assertEqual(res, ['start-ns', 'end-ns']) + it.close() + + def test_unknown_events(self): + iterparse = ET.iterparse + + events = ("start", "end", "bogus") + with open(SIMPLE_XMLFILE, "rb") as f: + with self.assertRaises(ValueError) as cm: + iterparse(f, events) + self.assertFalse(f.closed) + self.assertEqual(str(cm.exception), "unknown event 'bogus'") + + with warnings_helper.check_no_resource_warning(self): + with self.assertRaises(ValueError) as cm: + iterparse(SIMPLE_XMLFILE, events) + self.assertEqual(str(cm.exception), "unknown event 'bogus'") + del cm + gc_collect() + + def test_non_utf8(self): + source = io.BytesIO( + b"\n" + b"text\n") + events = ("start-ns",) + it = ET.iterparse(source, events) + self.assertEqual([(action, elem) for action, elem in it], [ + ('start-ns', ('', 'http://\xe9ffbot.org/ns')), + ('start-ns', ('cl\xe9', 'http://effbot.org/ns')), + ]) + + def test_parsing_error(self): + source = io.BytesIO(b"junk") + it = ET.iterparse(source) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'document')) + with self.assertRaises(ET.ParseError) as cm: + next(it) + self.assertEqual(str(cm.exception), + 'junk after document element: line 1, column 12') + + def test_nonexistent_file(self): + with self.assertRaises(FileNotFoundError): + ET.iterparse("nonexistent") + + def test_resource_warnings_not_exhausted(self): + # Not exhausting the iterator still closes the underlying file (bpo-43292) + it = ET.iterparse(SIMPLE_XMLFILE) + with warnings_helper.check_no_resource_warning(self): + del it + gc_collect() + + it = ET.iterparse(SIMPLE_XMLFILE) + with warnings_helper.check_no_resource_warning(self): + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + del it, elem + gc_collect() + + def test_resource_warnings_failed_iteration(self): + self.addCleanup(os_helper.unlink, TESTFN) + with open(TESTFN, "wb") as f: + f.write(b"junk") + + it = ET.iterparse(TESTFN) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'document')) + with warnings_helper.check_no_resource_warning(self): + with self.assertRaises(ET.ParseError) as cm: + next(it) + self.assertEqual(str(cm.exception), + 'junk after document element: line 1, column 12') + del cm, it + gc_collect() + + def test_resource_warnings_exhausted(self): + it = ET.iterparse(SIMPLE_XMLFILE) + with warnings_helper.check_no_resource_warning(self): + list(it) + del it + gc_collect() + + def test_close_not_exhausted(self): + iterparse = ET.iterparse + + it = iterparse(SIMPLE_XMLFILE) + it.close() + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + with open(SIMPLE_XMLFILE, 'rb') as source: + it = iterparse(source) + it.close() + self.assertFalse(source.closed) + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + it = iterparse(SIMPLE_XMLFILE) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + it.close() + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + with open(SIMPLE_XMLFILE, 'rb') as source: + it = iterparse(source) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + it.close() + self.assertFalse(source.closed) + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + def test_close_exhausted(self): + iterparse = ET.iterparse + it = iterparse(SIMPLE_XMLFILE) + list(it) + it.close() + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + with open(SIMPLE_XMLFILE, 'rb') as source: + it = iterparse(source) + list(it) + it.close() + self.assertFalse(source.closed) + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + class XMLPullParserTest(unittest.TestCase): def _feed(self, parser, data, chunk_size=None, flush=False): From 732224e1139f7ed4fe0259a2dad900f84910949e Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Thu, 13 Nov 2025 05:19:44 -0800 Subject: [PATCH 029/638] gh-139871: Add `bytearray.take_bytes([n])` to efficiently extract `bytes` (GH-140128) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update `bytearray` to contain a `bytes` and provide a zero-copy path to "extract" the `bytes`. This allows making several code paths more efficient. This does not move any codepaths to make use of this new API. The documentation changes include common code patterns which can be made more efficient with this API. --- When just changing `bytearray` to contain `bytes` I ran pyperformance on a `--with-lto --enable-optimizations --with-static-libpython` build and don't see any major speedups or slowdowns with this; all seems to be in the noise of my machine (Generally changes under 5% or benchmarks that don't touch bytes/bytearray). Co-authored-by: Victor Stinner Co-authored-by: Maurycy Pawłowski-Wieroński <5383+maurycy@users.noreply.github.com> --- Doc/library/stdtypes.rst | 24 ++ Doc/whatsnew/3.15.rst | 80 ++++++ Include/cpython/bytearrayobject.h | 16 +- Include/internal/pycore_bytesobject.h | 8 + Lib/test/test_bytes.py | 81 ++++++ Lib/test/test_capi/test_bytearray.py | 5 +- Lib/test/test_sys.py | 2 +- ...-10-14-18-24-16.gh-issue-139871.SWtuUz.rst | 2 + Objects/bytearrayobject.c | 236 ++++++++++++------ Objects/bytesobject.c | 8 +- Objects/clinic/bytearrayobject.c.h | 39 ++- 11 files changed, 406 insertions(+), 95 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-18-24-16.gh-issue-139871.SWtuUz.rst diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 97e7e08364e0bd..c539345e598777 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -3173,6 +3173,30 @@ objects. .. versionadded:: 3.14 + .. method:: take_bytes(n=None, /) + + Remove the first *n* bytes from the bytearray and return them as an immutable + :class:`bytes`. + By default (if *n* is ``None``), return all bytes and clear the bytearray. + + If *n* is negative, index from the end and take the first :func:`len` + plus *n* bytes. If *n* is out of bounds, raise :exc:`IndexError`. + + Taking less than the full length will leave remaining bytes in the + :class:`bytearray`, which requires a copy. If the remaining bytes should be + discarded, use :func:`~bytearray.resize` or :keyword:`del` to truncate + then :func:`~bytearray.take_bytes` without a size. + + .. impl-detail:: + + Taking all bytes is a zero-copy operation. + + .. versionadded:: next + + See the :ref:`What's New ` entry for + common code patterns which can be optimized with + :func:`bytearray.take_bytes`. + Since bytearray objects are sequences of integers (akin to a list), for a bytearray object *b*, ``b[0]`` will be an integer, while ``b[0:1]`` will be a bytearray object of length 1. (This contrasts with text strings, where diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 3cb766978a7217..d7c9a41eeb2759 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -307,6 +307,86 @@ Other language changes not only integers or floats, although this does not improve precision. (Contributed by Serhiy Storchaka in :gh:`67795`.) +.. _whatsnew315-bytearray-take-bytes: + +* Added :meth:`bytearray.take_bytes(n=None, /) ` to take + bytes out of a :class:`bytearray` without copying. This enables optimizing code + which must return :class:`bytes` after working with a mutable buffer of bytes + such as data buffering, network protocol parsing, encoding, decoding, + and compression. Common code patterns which can be optimized with + :func:`~bytearray.take_bytes` are listed below. + + (Contributed by Cody Maloney in :gh:`139871`.) + + .. list-table:: Suggested Optimizing Refactors + :header-rows: 1 + + * - Description + - Old + - New + + * - Return :class:`bytes` after working with :class:`bytearray` + - .. code:: python + + def read() -> bytes: + buffer = bytearray(1024) + ... + return bytes(buffer) + + - .. code:: python + + def read() -> bytes: + buffer = bytearray(1024) + ... + return buffer.take_bytes() + + * - Empty a buffer getting the bytes + - .. code:: python + + buffer = bytearray(1024) + ... + data = bytes(buffer) + buffer.clear() + + - .. code:: python + + buffer = bytearray(1024) + ... + data = buffer.take_bytes() + + * - Split a buffer at a specific separator + - .. code:: python + + buffer = bytearray(b'abc\ndef') + n = buffer.find(b'\n') + data = bytes(buffer[:n + 1]) + del buffer[:n + 1] + assert data == b'abc' + assert buffer == bytearray(b'def') + + - .. code:: python + + buffer = bytearray(b'abc\ndef') + n = buffer.find(b'\n') + data = buffer.take_bytes(n + 1) + + * - Split a buffer at a specific separator; discard after the separator + - .. code:: python + + buffer = bytearray(b'abc\ndef') + n = buffer.find(b'\n') + data = bytes(buffer[:n]) + buffer.clear() + assert data == b'abc' + assert len(buffer) == 0 + + - .. code:: python + + buffer = bytearray(b'abc\ndef') + n = buffer.find(b'\n') + buffer.resize(n) + data = buffer.take_bytes() + * Many functions related to compiling or parsing Python code, such as :func:`compile`, :func:`ast.parse`, :func:`symtable.symtable`, and :func:`importlib.abc.InspectLoader.source_to_code`, now allow to pass diff --git a/Include/cpython/bytearrayobject.h b/Include/cpython/bytearrayobject.h index 4dddef713ce097..1edd082074206c 100644 --- a/Include/cpython/bytearrayobject.h +++ b/Include/cpython/bytearrayobject.h @@ -5,25 +5,25 @@ /* Object layout */ typedef struct { PyObject_VAR_HEAD - Py_ssize_t ob_alloc; /* How many bytes allocated in ob_bytes */ + /* How many bytes allocated in ob_bytes + + In the current implementation this is equivalent to Py_SIZE(ob_bytes_object). + The value is always loaded and stored atomically for thread safety. + There are API compatibilty concerns with removing so keeping for now. */ + Py_ssize_t ob_alloc; char *ob_bytes; /* Physical backing buffer */ char *ob_start; /* Logical start inside ob_bytes */ Py_ssize_t ob_exports; /* How many buffer exports */ + PyObject *ob_bytes_object; /* PyBytes for zero-copy bytes conversion */ } PyByteArrayObject; -PyAPI_DATA(char) _PyByteArray_empty_string[]; - /* Macros and static inline functions, trading safety for speed */ #define _PyByteArray_CAST(op) \ (assert(PyByteArray_Check(op)), _Py_CAST(PyByteArrayObject*, op)) static inline char* PyByteArray_AS_STRING(PyObject *op) { - PyByteArrayObject *self = _PyByteArray_CAST(op); - if (Py_SIZE(self)) { - return self->ob_start; - } - return _PyByteArray_empty_string; + return _PyByteArray_CAST(op)->ob_start; } #define PyByteArray_AS_STRING(self) PyByteArray_AS_STRING(_PyObject_CAST(self)) diff --git a/Include/internal/pycore_bytesobject.h b/Include/internal/pycore_bytesobject.h index c7bc53b6073770..8e8fa696ee0350 100644 --- a/Include/internal/pycore_bytesobject.h +++ b/Include/internal/pycore_bytesobject.h @@ -60,6 +60,14 @@ PyAPI_FUNC(void) _PyBytes_Repeat(char* dest, Py_ssize_t len_dest, const char* src, Py_ssize_t len_src); +/* _PyBytesObject_SIZE gives the basic size of a bytes object; any memory allocation + for a bytes object of length n should request PyBytesObject_SIZE + n bytes. + + Using _PyBytesObject_SIZE instead of sizeof(PyBytesObject) saves + 3 or 7 bytes per bytes object allocation on a typical system. +*/ +#define _PyBytesObject_SIZE (offsetof(PyBytesObject, ob_sval) + 1) + /* --- PyBytesWriter ------------------------------------------------------ */ struct PyBytesWriter { diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index e012042159d223..86898bfcab9135 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1397,6 +1397,16 @@ def test_clear(self): b.append(ord('p')) self.assertEqual(b, b'p') + # Cleared object should be empty. + b = bytearray(b'abc') + b.clear() + self.assertEqual(b.__alloc__(), 0) + base_size = sys.getsizeof(bytearray()) + self.assertEqual(sys.getsizeof(b), base_size) + c = b.copy() + self.assertEqual(c.__alloc__(), 0) + self.assertEqual(sys.getsizeof(c), base_size) + def test_copy(self): b = bytearray(b'abc') bb = b.copy() @@ -1458,6 +1468,61 @@ def test_resize(self): self.assertRaises(MemoryError, bytearray().resize, sys.maxsize) self.assertRaises(MemoryError, bytearray(1000).resize, sys.maxsize) + def test_take_bytes(self): + ba = bytearray(b'ab') + self.assertEqual(ba.take_bytes(), b'ab') + self.assertEqual(len(ba), 0) + self.assertEqual(ba, bytearray(b'')) + self.assertEqual(ba.__alloc__(), 0) + base_size = sys.getsizeof(bytearray()) + self.assertEqual(sys.getsizeof(ba), base_size) + + # Positive and negative slicing. + ba = bytearray(b'abcdef') + self.assertEqual(ba.take_bytes(1), b'a') + self.assertEqual(ba, bytearray(b'bcdef')) + self.assertEqual(len(ba), 5) + self.assertEqual(ba.take_bytes(-5), b'') + self.assertEqual(ba, bytearray(b'bcdef')) + self.assertEqual(len(ba), 5) + self.assertEqual(ba.take_bytes(-3), b'bc') + self.assertEqual(ba, bytearray(b'def')) + self.assertEqual(len(ba), 3) + self.assertEqual(ba.take_bytes(3), b'def') + self.assertEqual(ba, bytearray(b'')) + self.assertEqual(len(ba), 0) + + # Take nothing from emptiness. + self.assertEqual(ba.take_bytes(0), b'') + self.assertEqual(ba.take_bytes(), b'') + self.assertEqual(ba.take_bytes(None), b'') + + # Out of bounds, bad take value. + self.assertRaises(IndexError, ba.take_bytes, -1) + self.assertRaises(TypeError, ba.take_bytes, 3.14) + ba = bytearray(b'abcdef') + self.assertRaises(IndexError, ba.take_bytes, 7) + + # Offset between physical and logical start (ob_bytes != ob_start). + ba = bytearray(b'abcde') + del ba[:2] + self.assertEqual(ba, bytearray(b'cde')) + self.assertEqual(ba.take_bytes(), b'cde') + + # Overallocation at end. + ba = bytearray(b'abcde') + del ba[-2:] + self.assertEqual(ba, bytearray(b'abc')) + self.assertEqual(ba.take_bytes(), b'abc') + ba = bytearray(b'abcde') + ba.resize(4) + self.assertEqual(ba.take_bytes(), b'abcd') + + # Take of a bytearray with references should fail. + ba = bytearray(b'abc') + with memoryview(ba) as mv: + self.assertRaises(BufferError, ba.take_bytes) + self.assertEqual(ba.take_bytes(), b'abc') def test_setitem(self): def setitem_as_mapping(b, i, val): @@ -2564,6 +2629,18 @@ def zfill(b, a): c = a.zfill(0x400000) assert not c or c[-1] not in (0xdd, 0xcd) + def take_bytes(b, a): # MODIFIES! + b.wait() + c = a.take_bytes() + assert not c or c[0] == 48 # '0' + + def take_bytes_n(b, a): # MODIFIES! + b.wait() + try: + c = a.take_bytes(10) + assert c == b'0123456789' + except IndexError: pass + def check(funcs, a=None, *args): if a is None: a = bytearray(b'0' * 0x400000) @@ -2625,6 +2702,10 @@ def check(funcs, a=None, *args): check([clear] + [startswith] * 10) check([clear] + [strip] * 10) + check([clear] + [take_bytes] * 10) + check([take_bytes_n] * 10, bytearray(b'0123456789' * 0x400)) + check([take_bytes_n] * 10, bytearray(b'0123456789' * 5)) + check([clear] + [contains] * 10) check([clear] + [subscript] * 10) check([clear2] + [ass_subscript2] * 10, None, bytearray(b'0' * 0x400000)) diff --git a/Lib/test/test_capi/test_bytearray.py b/Lib/test/test_capi/test_bytearray.py index 52565ea34c61b8..cb7ad8b22252d9 100644 --- a/Lib/test/test_capi/test_bytearray.py +++ b/Lib/test/test_capi/test_bytearray.py @@ -1,3 +1,4 @@ +import sys import unittest from test.support import import_helper @@ -55,7 +56,9 @@ def test_fromstringandsize(self): self.assertEqual(fromstringandsize(b'', 0), bytearray()) self.assertEqual(fromstringandsize(NULL, 0), bytearray()) self.assertEqual(len(fromstringandsize(NULL, 3)), 3) - self.assertRaises(MemoryError, fromstringandsize, NULL, PY_SSIZE_T_MAX) + self.assertRaises(OverflowError, fromstringandsize, NULL, PY_SSIZE_T_MAX) + self.assertRaises(OverflowError, fromstringandsize, NULL, + PY_SSIZE_T_MAX-sys.getsizeof(b'') + 1) self.assertRaises(SystemError, fromstringandsize, b'abc', -1) self.assertRaises(SystemError, fromstringandsize, b'abc', PY_SSIZE_T_MIN) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 3ceed019ac43cf..9d3248d972e8d1 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1583,7 +1583,7 @@ def test_objecttypes(self): samples = [b'', b'u'*100000] for sample in samples: x = bytearray(sample) - check(x, vsize('n2Pi') + x.__alloc__()) + check(x, vsize('n2PiP') + x.__alloc__()) # bytearray_iterator check(iter(bytearray()), size('nP')) # bytes diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-18-24-16.gh-issue-139871.SWtuUz.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-18-24-16.gh-issue-139871.SWtuUz.rst new file mode 100644 index 00000000000000..d4b8578afe3afc --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-18-24-16.gh-issue-139871.SWtuUz.rst @@ -0,0 +1,2 @@ +Update :class:`bytearray` to use a :class:`bytes` under the hood as its buffer +and add :func:`bytearray.take_bytes` to take it out. diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index a73bfff340ce48..99bfdec89f6c3a 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -17,8 +17,8 @@ class bytearray "PyByteArrayObject *" "&PyByteArray_Type" [clinic start generated code]*/ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=5535b77c37a119e0]*/ -/* For PyByteArray_AS_STRING(). */ -char _PyByteArray_empty_string[] = ""; +/* Max number of bytes a bytearray can contain */ +#define PyByteArray_SIZE_MAX ((Py_ssize_t)(PY_SSIZE_T_MAX - _PyBytesObject_SIZE)) /* Helpers */ @@ -43,6 +43,14 @@ _getbytevalue(PyObject* arg, int *value) return 1; } +static void +bytearray_reinit_from_bytes(PyByteArrayObject *self, Py_ssize_t size, + Py_ssize_t alloc) { + self->ob_bytes = self->ob_start = PyBytes_AS_STRING(self->ob_bytes_object); + Py_SET_SIZE(self, size); + FT_ATOMIC_STORE_SSIZE_RELAXED(self->ob_alloc, alloc); +} + static int bytearray_getbuffer_lock_held(PyObject *self, Py_buffer *view, int flags) { @@ -127,7 +135,6 @@ PyObject * PyByteArray_FromStringAndSize(const char *bytes, Py_ssize_t size) { PyByteArrayObject *new; - Py_ssize_t alloc; if (size < 0) { PyErr_SetString(PyExc_SystemError, @@ -135,34 +142,31 @@ PyByteArray_FromStringAndSize(const char *bytes, Py_ssize_t size) return NULL; } - /* Prevent buffer overflow when setting alloc to size+1. */ - if (size == PY_SSIZE_T_MAX) { - return PyErr_NoMemory(); - } - new = PyObject_New(PyByteArrayObject, &PyByteArray_Type); - if (new == NULL) + if (new == NULL) { return NULL; + } + + /* Fill values used in bytearray_dealloc. + + In an optimized build the memory isn't zeroed and ob_exports would be + uninitialized when when PyBytes_FromStringAndSize errored leading to + intermittent test failures. */ + new->ob_exports = 0; + + /* Optimization: size=0 bytearray should not allocate space - if (size == 0) { - new->ob_bytes = NULL; - alloc = 0; + PyBytes_FromStringAndSize returns the empty bytes global when size=0 so + no allocation occurs. */ + new->ob_bytes_object = PyBytes_FromStringAndSize(NULL, size); + if (new->ob_bytes_object == NULL) { + Py_DECREF(new); + return NULL; } - else { - alloc = size + 1; - new->ob_bytes = PyMem_Malloc(alloc); - if (new->ob_bytes == NULL) { - Py_DECREF(new); - return PyErr_NoMemory(); - } - if (bytes != NULL && size > 0) - memcpy(new->ob_bytes, bytes, size); - new->ob_bytes[size] = '\0'; /* Trailing null byte */ + bytearray_reinit_from_bytes(new, size, size); + if (bytes != NULL && size > 0) { + memcpy(new->ob_bytes, bytes, size); } - Py_SET_SIZE(new, size); - new->ob_alloc = alloc; - new->ob_start = new->ob_bytes; - new->ob_exports = 0; return (PyObject *)new; } @@ -189,7 +193,6 @@ static int bytearray_resize_lock_held(PyObject *self, Py_ssize_t requested_size) { _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); - void *sval; PyByteArrayObject *obj = ((PyByteArrayObject *)self); /* All computations are done unsigned to avoid integer overflows (see issue #22335). */ @@ -214,16 +217,17 @@ bytearray_resize_lock_held(PyObject *self, Py_ssize_t requested_size) return -1; } - if (size + logical_offset + 1 <= alloc) { + if (size + logical_offset <= alloc) { /* Current buffer is large enough to host the requested size, decide on a strategy. */ if (size < alloc / 2) { /* Major downsize; resize down to exact size */ - alloc = size + 1; + alloc = size; } else { /* Minor downsize; quick exit */ Py_SET_SIZE(self, size); + /* Add mid-buffer null; end provided by bytes. */ PyByteArray_AS_STRING(self)[size] = '\0'; /* Trailing null */ return 0; } @@ -236,38 +240,36 @@ bytearray_resize_lock_held(PyObject *self, Py_ssize_t requested_size) } else { /* Major upsize; resize up to exact size */ - alloc = size + 1; + alloc = size; } } - if (alloc > PY_SSIZE_T_MAX) { + if (alloc > PyByteArray_SIZE_MAX) { PyErr_NoMemory(); return -1; } + /* Re-align data to the start of the allocation. */ if (logical_offset > 0) { - sval = PyMem_Malloc(alloc); - if (sval == NULL) { - PyErr_NoMemory(); - return -1; - } - memcpy(sval, PyByteArray_AS_STRING(self), - Py_MIN((size_t)requested_size, (size_t)Py_SIZE(self))); - PyMem_Free(obj->ob_bytes); - } - else { - sval = PyMem_Realloc(obj->ob_bytes, alloc); - if (sval == NULL) { - PyErr_NoMemory(); - return -1; - } + /* optimization tradeoff: This is faster than a new allocation when + the number of bytes being removed in a resize is small; for large + size changes it may be better to just make a new bytes object as + _PyBytes_Resize will do a malloc + memcpy internally. */ + memmove(obj->ob_bytes, obj->ob_start, + Py_MIN(requested_size, Py_SIZE(self))); } - obj->ob_bytes = obj->ob_start = sval; - Py_SET_SIZE(self, size); - FT_ATOMIC_STORE_SSIZE_RELAXED(obj->ob_alloc, alloc); - obj->ob_bytes[size] = '\0'; /* Trailing null byte */ + int ret = _PyBytes_Resize(&obj->ob_bytes_object, alloc); + if (ret == -1) { + obj->ob_bytes_object = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); + size = alloc = 0; + } + bytearray_reinit_from_bytes(obj, size, alloc); + if (alloc != size) { + /* Add mid-buffer null; end provided by bytes. */ + obj->ob_bytes[size] = '\0'; + } - return 0; + return ret; } int @@ -295,7 +297,7 @@ PyByteArray_Concat(PyObject *a, PyObject *b) goto done; } - if (va.len > PY_SSIZE_T_MAX - vb.len) { + if (va.len > PyByteArray_SIZE_MAX - vb.len) { PyErr_NoMemory(); goto done; } @@ -339,7 +341,7 @@ bytearray_iconcat_lock_held(PyObject *op, PyObject *other) } Py_ssize_t size = Py_SIZE(self); - if (size > PY_SSIZE_T_MAX - vo.len) { + if (size > PyByteArray_SIZE_MAX - vo.len) { PyBuffer_Release(&vo); return PyErr_NoMemory(); } @@ -373,7 +375,7 @@ bytearray_repeat_lock_held(PyObject *op, Py_ssize_t count) count = 0; } const Py_ssize_t mysize = Py_SIZE(self); - if (count > 0 && mysize > PY_SSIZE_T_MAX / count) { + if (count > 0 && mysize > PyByteArray_SIZE_MAX / count) { return PyErr_NoMemory(); } Py_ssize_t size = mysize * count; @@ -409,7 +411,7 @@ bytearray_irepeat_lock_held(PyObject *op, Py_ssize_t count) } const Py_ssize_t mysize = Py_SIZE(self); - if (count > 0 && mysize > PY_SSIZE_T_MAX / count) { + if (count > 0 && mysize > PyByteArray_SIZE_MAX / count) { return PyErr_NoMemory(); } const Py_ssize_t size = mysize * count; @@ -585,7 +587,7 @@ bytearray_setslice_linear(PyByteArrayObject *self, buf = PyByteArray_AS_STRING(self); } else if (growth > 0) { - if (Py_SIZE(self) > (Py_ssize_t)PY_SSIZE_T_MAX - growth) { + if (Py_SIZE(self) > PyByteArray_SIZE_MAX - growth) { PyErr_NoMemory(); return -1; } @@ -899,6 +901,13 @@ bytearray___init___impl(PyByteArrayObject *self, PyObject *arg, PyObject *it; PyObject *(*iternext)(PyObject *); + /* First __init__; set ob_bytes_object so ob_bytes is always non-null. */ + if (self->ob_bytes_object == NULL) { + self->ob_bytes_object = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); + bytearray_reinit_from_bytes(self, 0, 0); + self->ob_exports = 0; + } + if (Py_SIZE(self) != 0) { /* Empty previous contents (yes, do this first of all!) */ if (PyByteArray_Resize((PyObject *)self, 0) < 0) @@ -1169,9 +1178,7 @@ bytearray_dealloc(PyObject *op) "deallocated bytearray object has exported buffers"); PyErr_Print(); } - if (self->ob_bytes != 0) { - PyMem_Free(self->ob_bytes); - } + Py_XDECREF(self->ob_bytes_object); Py_TYPE(self)->tp_free((PyObject *)self); } @@ -1491,6 +1498,82 @@ bytearray_resize_impl(PyByteArrayObject *self, Py_ssize_t size) } +/*[clinic input] +@critical_section +bytearray.take_bytes + n: object = None + Bytes to take, negative indexes from end. None indicates all bytes. + / +Take *n* bytes from the bytearray and return them as a bytes object. +[clinic start generated code]*/ + +static PyObject * +bytearray_take_bytes_impl(PyByteArrayObject *self, PyObject *n) +/*[clinic end generated code: output=3147fbc0bbbe8d94 input=b15b5172cdc6deda]*/ +{ + Py_ssize_t to_take; + Py_ssize_t size = Py_SIZE(self); + if (Py_IsNone(n)) { + to_take = size; + } + // Integer index, from start (zero, positive) or end (negative). + else if (_PyIndex_Check(n)) { + to_take = PyNumber_AsSsize_t(n, PyExc_IndexError); + if (to_take == -1 && PyErr_Occurred()) { + return NULL; + } + if (to_take < 0) { + to_take += size; + } + } + else { + PyErr_SetString(PyExc_TypeError, "n must be an integer or None"); + return NULL; + } + + if (to_take < 0 || to_take > size) { + PyErr_Format(PyExc_IndexError, + "can't take %zd bytes outside size %zd", + to_take, size); + return NULL; + } + + // Exports may change the contents. No mutable bytes allowed. + if (!_canresize(self)) { + return NULL; + } + + if (to_take == 0 || size == 0) { + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); + } + + // Copy remaining bytes to a new bytes. + Py_ssize_t remaining_length = size - to_take; + PyObject *remaining = PyBytes_FromStringAndSize(self->ob_start + to_take, + remaining_length); + if (remaining == NULL) { + return NULL; + } + + // If the bytes are offset inside the buffer must first align. + if (self->ob_start != self->ob_bytes) { + memmove(self->ob_bytes, self->ob_start, to_take); + self->ob_start = self->ob_bytes; + } + + if (_PyBytes_Resize(&self->ob_bytes_object, to_take) == -1) { + Py_DECREF(remaining); + return NULL; + } + + // Point the bytearray towards the buffer with the remaining data. + PyObject *result = self->ob_bytes_object; + self->ob_bytes_object = remaining; + bytearray_reinit_from_bytes(self, remaining_length, remaining_length); + return result; +} + + /*[clinic input] @critical_section bytearray.translate @@ -1868,11 +1951,6 @@ bytearray_insert_impl(PyByteArrayObject *self, Py_ssize_t index, int item) Py_ssize_t n = Py_SIZE(self); char *buf; - if (n == PY_SSIZE_T_MAX) { - PyErr_SetString(PyExc_OverflowError, - "cannot add more objects to bytearray"); - return NULL; - } if (bytearray_resize_lock_held((PyObject *)self, n + 1) < 0) return NULL; buf = PyByteArray_AS_STRING(self); @@ -1987,11 +2065,6 @@ bytearray_append_impl(PyByteArrayObject *self, int item) { Py_ssize_t n = Py_SIZE(self); - if (n == PY_SSIZE_T_MAX) { - PyErr_SetString(PyExc_OverflowError, - "cannot add more objects to bytearray"); - return NULL; - } if (bytearray_resize_lock_held((PyObject *)self, n + 1) < 0) return NULL; @@ -2099,16 +2172,16 @@ bytearray_extend_impl(PyByteArrayObject *self, PyObject *iterable_of_ints) if (len >= buf_size) { Py_ssize_t addition; - if (len == PY_SSIZE_T_MAX) { + if (len == PyByteArray_SIZE_MAX) { Py_DECREF(it); Py_DECREF(bytearray_obj); return PyErr_NoMemory(); } addition = len >> 1; - if (addition > PY_SSIZE_T_MAX - len - 1) - buf_size = PY_SSIZE_T_MAX; + if (addition > PyByteArray_SIZE_MAX - len) + buf_size = PyByteArray_SIZE_MAX; else - buf_size = len + addition + 1; + buf_size = len + addition; if (bytearray_resize_lock_held((PyObject *)bytearray_obj, buf_size) < 0) { Py_DECREF(it); Py_DECREF(bytearray_obj); @@ -2405,7 +2478,11 @@ static PyObject * bytearray_alloc(PyObject *op, PyObject *Py_UNUSED(ignored)) { PyByteArrayObject *self = _PyByteArray_CAST(op); - return PyLong_FromSsize_t(FT_ATOMIC_LOAD_SSIZE_RELAXED(self->ob_alloc)); + Py_ssize_t alloc = FT_ATOMIC_LOAD_SSIZE_RELAXED(self->ob_alloc); + if (alloc > 0) { + alloc += _PyBytesObject_SIZE; + } + return PyLong_FromSsize_t(alloc); } /*[clinic input] @@ -2601,9 +2678,13 @@ static PyObject * bytearray_sizeof_impl(PyByteArrayObject *self) /*[clinic end generated code: output=738abdd17951c427 input=e27320fd98a4bc5a]*/ { - size_t res = _PyObject_SIZE(Py_TYPE(self)); - res += (size_t)FT_ATOMIC_LOAD_SSIZE_RELAXED(self->ob_alloc) * sizeof(char); - return PyLong_FromSize_t(res); + Py_ssize_t res = _PyObject_SIZE(Py_TYPE(self)); + Py_ssize_t alloc = FT_ATOMIC_LOAD_SSIZE_RELAXED(self->ob_alloc); + if (alloc > 0) { + res += _PyBytesObject_SIZE + alloc; + } + + return PyLong_FromSsize_t(res); } static PySequenceMethods bytearray_as_sequence = { @@ -2686,6 +2767,7 @@ static PyMethodDef bytearray_methods[] = { BYTEARRAY_STARTSWITH_METHODDEF BYTEARRAY_STRIP_METHODDEF {"swapcase", bytearray_swapcase, METH_NOARGS, _Py_swapcase__doc__}, + BYTEARRAY_TAKE_BYTES_METHODDEF {"title", bytearray_title, METH_NOARGS, _Py_title__doc__}, BYTEARRAY_TRANSLATE_METHODDEF {"upper", bytearray_upper, METH_NOARGS, _Py_upper__doc__}, diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 2b9513abe91956..2b0925017f29e4 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -25,13 +25,7 @@ class bytes "PyBytesObject *" "&PyBytes_Type" #include "clinic/bytesobject.c.h" -/* PyBytesObject_SIZE gives the basic size of a bytes object; any memory allocation - for a bytes object of length n should request PyBytesObject_SIZE + n bytes. - - Using PyBytesObject_SIZE instead of sizeof(PyBytesObject) saves - 3 or 7 bytes per bytes object allocation on a typical system. -*/ -#define PyBytesObject_SIZE (offsetof(PyBytesObject, ob_sval) + 1) +#define PyBytesObject_SIZE _PyBytesObject_SIZE /* Forward declaration */ static void* _PyBytesWriter_ResizeAndUpdatePointer(PyBytesWriter *writer, diff --git a/Objects/clinic/bytearrayobject.c.h b/Objects/clinic/bytearrayobject.c.h index 6f13865177dde5..be704ccf68f669 100644 --- a/Objects/clinic/bytearrayobject.c.h +++ b/Objects/clinic/bytearrayobject.c.h @@ -631,6 +631,43 @@ bytearray_resize(PyObject *self, PyObject *arg) return return_value; } +PyDoc_STRVAR(bytearray_take_bytes__doc__, +"take_bytes($self, n=None, /)\n" +"--\n" +"\n" +"Take *n* bytes from the bytearray and return them as a bytes object.\n" +"\n" +" n\n" +" Bytes to take, negative indexes from end. None indicates all bytes."); + +#define BYTEARRAY_TAKE_BYTES_METHODDEF \ + {"take_bytes", _PyCFunction_CAST(bytearray_take_bytes), METH_FASTCALL, bytearray_take_bytes__doc__}, + +static PyObject * +bytearray_take_bytes_impl(PyByteArrayObject *self, PyObject *n); + +static PyObject * +bytearray_take_bytes(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *n = Py_None; + + if (!_PyArg_CheckPositional("take_bytes", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + n = args[0]; +skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); + return_value = bytearray_take_bytes_impl((PyByteArrayObject *)self, n); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + PyDoc_STRVAR(bytearray_translate__doc__, "translate($self, table, /, delete=b\'\')\n" "--\n" @@ -1796,4 +1833,4 @@ bytearray_sizeof(PyObject *self, PyObject *Py_UNUSED(ignored)) { return bytearray_sizeof_impl((PyByteArrayObject *)self); } -/*[clinic end generated code: output=fdfe41139c91e409 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5eddefde2a001ceb input=a9049054013a1b77]*/ From c2470b39fa21f355f811419f1b3d1c776c36fb10 Mon Sep 17 00:00:00 2001 From: Mikhail Efimov Date: Thu, 13 Nov 2025 17:44:40 +0300 Subject: [PATCH 030/638] gh-137959: Fix `TIER1_TO_TIER2` macro name in JIT InternalDocs (GH-141496) JIT InternalDocs fix --- InternalDocs/jit.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/InternalDocs/jit.md b/InternalDocs/jit.md index 095853807377a4..1740b22b85f77b 100644 --- a/InternalDocs/jit.md +++ b/InternalDocs/jit.md @@ -53,7 +53,7 @@ and an instance of `_PyUOpExecutor_Type` is created to contain it. ## The JIT interpreter After a `JUMP_BACKWARD` instruction invokes the uop optimizer to create a uop -executor, it transfers control to this executor via the `GOTO_TIER_TWO` macro. +executor, it transfers control to this executor via the `TIER1_TO_TIER2` macro. CPython implements two executors. Here we describe the JIT interpreter, which is the simpler of them and is therefore useful for debugging and analyzing From f72768f30e6ed9253eb3b6374b4395dfcaf4842a Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 13 Nov 2025 10:02:21 -0500 Subject: [PATCH 031/638] gh-141004: Document C APIs for dictionary keys, values, and items (GH-141009) Co-authored-by: Petr Viktorin --- Doc/c-api/dict.rst | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index 246ce5391e142c..b7f201811aad6c 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -431,3 +431,49 @@ Dictionary Objects it before returning. .. versionadded:: 3.12 + + +Dictionary View Objects +^^^^^^^^^^^^^^^^^^^^^^^ + +.. c:function:: int PyDictViewSet_Check(PyObject *op) + + Return true if *op* is a view of a set inside a dictionary. This is currently + equivalent to :c:expr:`PyDictKeys_Check(op) || PyDictItems_Check(op)`. This + function always succeeds. + + +.. c:var:: PyTypeObject PyDictKeys_Type + + Type object for a view of dictionary keys. In Python, this is the type of + the object returned by :meth:`dict.keys`. + + +.. c:function:: int PyDictKeys_Check(PyObject *op) + + Return true if *op* is an instance of a dictionary keys view. This function + always succeeds. + + +.. c:var:: PyTypeObject PyDictValues_Type + + Type object for a view of dictionary values. In Python, this is the type of + the object returned by :meth:`dict.values`. + + +.. c:function:: int PyDictValues_Check(PyObject *op) + + Return true if *op* is an instance of a dictionary values view. This function + always succeeds. + + +.. c:var:: PyTypeObject PyDictItems_Type + + Type object for a view of dictionary items. In Python, this is the type of + the object returned by :meth:`dict.items`. + + +.. c:function:: int PyDictItems_Check(PyObject *op) + + Return true if *op* is an instance of a dictionary items view. This function + always succeeds. From d7862e9b1bd8f82e41c4f2c4dad31e15707d856f Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Thu, 13 Nov 2025 10:07:57 -0500 Subject: [PATCH 032/638] gh-141004: Document `PyCode_Optimize` (GH-141378) --- Doc/c-api/code.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index c9741b61254b19..45f5e83adc48c6 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -211,6 +211,17 @@ bound into a function. .. versionadded:: 3.12 +.. c:function:: PyObject *PyCode_Optimize(PyObject *code, PyObject *consts, PyObject *names, PyObject *lnotab_obj) + + This is a :term:`soft deprecated` function that does nothing. + + Prior to Python 3.10, this function would perform basic optimizations to a + code object. + + .. versionchanged:: 3.10 + This function now does nothing. + + .. _c_codeobject_flags: Code Object Flags From b99db92dde38b17c3fba3b5db76a383ceddfce49 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 13 Nov 2025 17:30:50 +0100 Subject: [PATCH 033/638] gh-139653: Add PyUnstable_ThreadState_SetStackProtection() (#139668) Add PyUnstable_ThreadState_SetStackProtection() and PyUnstable_ThreadState_ResetStackProtection() functions to set the stack base address and stack size of a Python thread state. Co-authored-by: Petr Viktorin --- Doc/c-api/exceptions.rst | 3 + Doc/c-api/init.rst | 37 +++++++++ Doc/whatsnew/3.15.rst | 6 ++ Include/cpython/pystate.h | 12 +++ Include/internal/pycore_pythonrun.h | 6 ++ Include/internal/pycore_tstate.h | 4 + ...-10-06-22-17-47.gh-issue-139653.6-1MOd.rst | 4 + Modules/_testinternalcapi.c | 54 +++++++++++++ Python/ceval.c | 77 +++++++++++++++++-- Python/pystate.c | 3 + 10 files changed, 199 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-10-06-22-17-47.gh-issue-139653.6-1MOd.rst diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 5241533e11281f..0ee595a07acc77 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -976,6 +976,9 @@ because the :ref:`call protocol ` takes care of recursion handling. be concatenated to the :exc:`RecursionError` message caused by the recursion depth limit. + .. seealso:: + The :c:func:`PyUnstable_ThreadState_SetStackProtection` function. + .. versionchanged:: 3.9 This function is now also available in the :ref:`limited API `. diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 49ffeab55850c0..18ee16118070eb 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -1366,6 +1366,43 @@ All of the following functions must be called after :c:func:`Py_Initialize`. .. versionadded:: 3.11 +.. c:function:: int PyUnstable_ThreadState_SetStackProtection(PyThreadState *tstate, void *stack_start_addr, size_t stack_size) + + Set the stack protection start address and stack protection size + of a Python thread state. + + On success, return ``0``. + On failure, set an exception and return ``-1``. + + CPython implements :ref:`recursion control ` for C code by raising + :py:exc:`RecursionError` when it notices that the machine execution stack is close + to overflow. See for example the :c:func:`Py_EnterRecursiveCall` function. + For this, it needs to know the location of the current thread's stack, which it + normally gets from the operating system. + When the stack is changed, for example using context switching techniques like the + Boost library's ``boost::context``, you must call + :c:func:`~PyUnstable_ThreadState_SetStackProtection` to inform CPython of the change. + + Call :c:func:`~PyUnstable_ThreadState_SetStackProtection` either before + or after changing the stack. + Do not call any other Python C API between the call and the stack + change. + + See :c:func:`PyUnstable_ThreadState_ResetStackProtection` for undoing this operation. + + .. versionadded:: next + + +.. c:function:: void PyUnstable_ThreadState_ResetStackProtection(PyThreadState *tstate) + + Reset the stack protection start address and stack protection size + of a Python thread state to the operating system defaults. + + See :c:func:`PyUnstable_ThreadState_SetStackProtection` for an explanation. + + .. versionadded:: next + + .. c:function:: PyInterpreterState* PyInterpreterState_Get(void) Get the current interpreter. diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index d7c9a41eeb2759..b360ad964cf17f 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -1066,6 +1066,12 @@ New features * Add :c:func:`PyTuple_FromArray` to create a :class:`tuple` from an array. (Contributed by Victor Stinner in :gh:`111489`.) +* Add :c:func:`PyUnstable_ThreadState_SetStackProtection` and + :c:func:`PyUnstable_ThreadState_ResetStackProtection` functions to set + the stack protection base address and stack protection size of a Python + thread state. + (Contributed by Victor Stinner in :gh:`139653`.) + Changed C APIs -------------- diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index dd2ea1202b3795..c53abe43ebe65c 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -276,6 +276,18 @@ PyAPI_FUNC(int) PyGILState_Check(void); */ PyAPI_FUNC(PyObject*) _PyThread_CurrentFrames(void); +// Set the stack protection start address and stack protection size +// of a Python thread state +PyAPI_FUNC(int) PyUnstable_ThreadState_SetStackProtection( + PyThreadState *tstate, + void *stack_start_addr, // Stack start address + size_t stack_size); // Stack size (in bytes) + +// Reset the stack protection start address and stack protection size +// of a Python thread state +PyAPI_FUNC(void) PyUnstable_ThreadState_ResetStackProtection( + PyThreadState *tstate); + /* Routines for advanced debuggers, requested by David Beazley. Don't use unless you know what you are doing! */ PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_Main(void); diff --git a/Include/internal/pycore_pythonrun.h b/Include/internal/pycore_pythonrun.h index f954f1b63ef67c..04a557e1204064 100644 --- a/Include/internal/pycore_pythonrun.h +++ b/Include/internal/pycore_pythonrun.h @@ -60,6 +60,12 @@ extern PyObject * _Py_CompileStringObjectWithModule( # define _PyOS_STACK_MARGIN_SHIFT (_PyOS_LOG2_STACK_MARGIN + 2) #endif +#ifdef _Py_THREAD_SANITIZER +# define _PyOS_MIN_STACK_SIZE (_PyOS_STACK_MARGIN_BYTES * 6) +#else +# define _PyOS_MIN_STACK_SIZE (_PyOS_STACK_MARGIN_BYTES * 3) +#endif + #ifdef __cplusplus } diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index 29ebdfd7e01613..a44c523e2022a7 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -37,6 +37,10 @@ typedef struct _PyThreadStateImpl { uintptr_t c_stack_soft_limit; uintptr_t c_stack_hard_limit; + // PyUnstable_ThreadState_ResetStackProtection() values + uintptr_t c_stack_init_base; + uintptr_t c_stack_init_top; + PyObject *asyncio_running_loop; // Strong reference PyObject *asyncio_running_task; // Strong reference diff --git a/Misc/NEWS.d/next/C_API/2025-10-06-22-17-47.gh-issue-139653.6-1MOd.rst b/Misc/NEWS.d/next/C_API/2025-10-06-22-17-47.gh-issue-139653.6-1MOd.rst new file mode 100644 index 00000000000000..cd3d5262fa0f3a --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-10-06-22-17-47.gh-issue-139653.6-1MOd.rst @@ -0,0 +1,4 @@ +Add :c:func:`PyUnstable_ThreadState_SetStackProtection` and +:c:func:`PyUnstable_ThreadState_ResetStackProtection` functions to set the +stack protection base address and stack protection size of a Python thread +state. Patch by Victor Stinner. diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index dede05960d78b6..6514ca7f3cd6de 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -2446,6 +2446,58 @@ module_get_gc_hooks(PyObject *self, PyObject *arg) return result; } + +static void +check_threadstate_set_stack_protection(PyThreadState *tstate, + void *start, size_t size) +{ + assert(PyUnstable_ThreadState_SetStackProtection(tstate, start, size) == 0); + assert(!PyErr_Occurred()); + + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; + assert(ts->c_stack_top == (uintptr_t)start + size); + assert(ts->c_stack_hard_limit <= ts->c_stack_soft_limit); + assert(ts->c_stack_soft_limit < ts->c_stack_top); +} + + +static PyObject * +test_threadstate_set_stack_protection(PyObject *self, PyObject *Py_UNUSED(args)) +{ + PyThreadState *tstate = PyThreadState_GET(); + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; + assert(!PyErr_Occurred()); + + uintptr_t init_base = ts->c_stack_init_base; + size_t init_top = ts->c_stack_init_top; + + // Test the minimum stack size + size_t size = _PyOS_MIN_STACK_SIZE; + void *start = (void*)(_Py_get_machine_stack_pointer() - size); + check_threadstate_set_stack_protection(tstate, start, size); + + // Test a larger size + size = 7654321; + assert(size > _PyOS_MIN_STACK_SIZE); + start = (void*)(_Py_get_machine_stack_pointer() - size); + check_threadstate_set_stack_protection(tstate, start, size); + + // Test invalid size (too small) + size = 5; + start = (void*)(_Py_get_machine_stack_pointer() - size); + assert(PyUnstable_ThreadState_SetStackProtection(tstate, start, size) == -1); + assert(PyErr_ExceptionMatches(PyExc_ValueError)); + PyErr_Clear(); + + // Test PyUnstable_ThreadState_ResetStackProtection() + PyUnstable_ThreadState_ResetStackProtection(tstate); + assert(ts->c_stack_init_base == init_base); + assert(ts->c_stack_init_top == init_top); + + Py_RETURN_NONE; +} + + static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, @@ -2556,6 +2608,8 @@ static PyMethodDef module_functions[] = { {"simple_pending_call", simple_pending_call, METH_O}, {"set_vectorcall_nop", set_vectorcall_nop, METH_O}, {"module_get_gc_hooks", module_get_gc_hooks, METH_O}, + {"test_threadstate_set_stack_protection", + test_threadstate_set_stack_protection, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Python/ceval.c b/Python/ceval.c index 43e8ee71206566..07d21575e3a266 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -443,7 +443,7 @@ int pthread_attr_destroy(pthread_attr_t *a) #endif static void -hardware_stack_limits(uintptr_t *top, uintptr_t *base) +hardware_stack_limits(uintptr_t *base, uintptr_t *top) { #ifdef WIN32 ULONG_PTR low, high; @@ -486,23 +486,86 @@ hardware_stack_limits(uintptr_t *top, uintptr_t *base) #endif } -void -_Py_InitializeRecursionLimits(PyThreadState *tstate) +static void +tstate_set_stack(PyThreadState *tstate, + uintptr_t base, uintptr_t top) { - uintptr_t top; - uintptr_t base; - hardware_stack_limits(&top, &base); + assert(base < top); + assert((top - base) >= _PyOS_MIN_STACK_SIZE); + #ifdef _Py_THREAD_SANITIZER // Thread sanitizer crashes if we use more than half the stack. uintptr_t stacksize = top - base; - base += stacksize/2; + base += stacksize / 2; #endif _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; _tstate->c_stack_top = top; _tstate->c_stack_hard_limit = base + _PyOS_STACK_MARGIN_BYTES; _tstate->c_stack_soft_limit = base + _PyOS_STACK_MARGIN_BYTES * 2; + +#ifndef NDEBUG + // Sanity checks + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; + assert(ts->c_stack_hard_limit <= ts->c_stack_soft_limit); + assert(ts->c_stack_soft_limit < ts->c_stack_top); +#endif +} + + +void +_Py_InitializeRecursionLimits(PyThreadState *tstate) +{ + uintptr_t base, top; + hardware_stack_limits(&base, &top); + assert(top != 0); + + tstate_set_stack(tstate, base, top); + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; + ts->c_stack_init_base = base; + ts->c_stack_init_top = top; + + // Test the stack pointer +#if !defined(NDEBUG) && !defined(__wasi__) + uintptr_t here_addr = _Py_get_machine_stack_pointer(); + assert(ts->c_stack_soft_limit < here_addr); + assert(here_addr < ts->c_stack_top); +#endif +} + + +int +PyUnstable_ThreadState_SetStackProtection(PyThreadState *tstate, + void *stack_start_addr, size_t stack_size) +{ + if (stack_size < _PyOS_MIN_STACK_SIZE) { + PyErr_Format(PyExc_ValueError, + "stack_size must be at least %zu bytes", + _PyOS_MIN_STACK_SIZE); + return -1; + } + + uintptr_t base = (uintptr_t)stack_start_addr; + uintptr_t top = base + stack_size; + tstate_set_stack(tstate, base, top); + return 0; } + +void +PyUnstable_ThreadState_ResetStackProtection(PyThreadState *tstate) +{ + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; + if (ts->c_stack_init_top != 0) { + tstate_set_stack(tstate, + ts->c_stack_init_base, + ts->c_stack_init_top); + return; + } + + _Py_InitializeRecursionLimits(tstate); +} + + /* The function _Py_EnterRecursiveCallTstate() only calls _Py_CheckRecursiveCall() if the recursion_depth reaches recursion_limit. */ int diff --git a/Python/pystate.c b/Python/pystate.c index cf251c120d75af..341c680a403608 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1495,6 +1495,9 @@ init_threadstate(_PyThreadStateImpl *_tstate, _tstate->c_stack_top = 0; _tstate->c_stack_hard_limit = 0; + _tstate->c_stack_init_base = 0; + _tstate->c_stack_init_top = 0; + _tstate->asyncio_running_loop = NULL; _tstate->asyncio_running_task = NULL; From b2b68d40f887c8a9583a9b48babc40f25bc5e0e2 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 13 Nov 2025 19:48:52 +0200 Subject: [PATCH 034/638] gh-140873: Add support of non-descriptor callables in functools.singledispatchmethod() (GH-140884) --- Doc/library/functools.rst | 5 ++- Doc/whatsnew/3.15.rst | 8 +++++ Lib/functools.py | 5 ++- Lib/test/test_functools.py | 35 ++++++++++++++++++- ...-11-01-14-44-09.gh-issue-140873.kfuc9B.rst | 2 ++ 5 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-01-14-44-09.gh-issue-140873.kfuc9B.rst diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 1d9ac328f32769..b2e2e11c0dc414 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -672,7 +672,7 @@ The :mod:`functools` module defines the following functions: dispatch>` :term:`generic function`. To define a generic method, decorate it with the ``@singledispatchmethod`` - decorator. When defining a function using ``@singledispatchmethod``, note + decorator. When defining a method using ``@singledispatchmethod``, note that the dispatch happens on the type of the first non-*self* or non-*cls* argument:: @@ -716,6 +716,9 @@ The :mod:`functools` module defines the following functions: .. versionadded:: 3.8 + .. versionchanged:: next + Added support of non-:term:`descriptor` callables. + .. function:: update_wrapper(wrapper, wrapped, assigned=WRAPPER_ASSIGNMENTS, updated=WRAPPER_UPDATES) diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index b360ad964cf17f..895616e3049a50 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -498,6 +498,14 @@ difflib (Contributed by Jiahao Li in :gh:`134580`.) +functools +--------- + +* :func:`~functools.singledispatchmethod` now supports non-:term:`descriptor` + callables. + (Contributed by Serhiy Storchaka in :gh:`140873`.) + + hashlib ------- diff --git a/Lib/functools.py b/Lib/functools.py index a92844ba7227b0..8063eb5ffc3304 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -1083,7 +1083,10 @@ def __call__(self, /, *args, **kwargs): 'singledispatchmethod method') raise TypeError(f'{funcname} requires at least ' '1 positional argument') - return self._dispatch(args[0].__class__).__get__(self._obj, self._cls)(*args, **kwargs) + method = self._dispatch(args[0].__class__) + if hasattr(method, "__get__"): + method = method.__get__(self._obj, self._cls) + return method(*args, **kwargs) def __getattr__(self, name): # Resolve these attributes lazily to speed up creation of diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index ce9e7f6d57dd3c..090926fd8d8b61 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -2785,7 +2785,7 @@ class Slot: @functools.singledispatchmethod @classmethod def go(cls, item, arg): - pass + return item - arg @go.register @classmethod @@ -2794,7 +2794,9 @@ def _(cls, item: int, arg): s = Slot() self.assertEqual(s.go(1, 1), 2) + self.assertEqual(s.go(1.5, 1), 0.5) self.assertEqual(Slot.go(1, 1), 2) + self.assertEqual(Slot.go(1.5, 1), 0.5) def test_staticmethod_slotted_class(self): class A: @@ -3485,6 +3487,37 @@ def _(item, arg: bytes) -> str: self.assertEqual(str(Signature.from_callable(A.static_func)), '(item, arg: int) -> str') + def test_method_non_descriptor(self): + class Callable: + def __init__(self, value): + self.value = value + def __call__(self, arg): + return self.value, arg + + class A: + t = functools.singledispatchmethod(Callable('general')) + t.register(int, Callable('special')) + + @functools.singledispatchmethod + def u(self, arg): + return 'general', arg + u.register(int, Callable('special')) + + v = functools.singledispatchmethod(Callable('general')) + @v.register(int) + def _(self, arg): + return 'special', arg + + a = A() + self.assertEqual(a.t(0), ('special', 0)) + self.assertEqual(a.t(2.5), ('general', 2.5)) + self.assertEqual(A.t(0), ('special', 0)) + self.assertEqual(A.t(2.5), ('general', 2.5)) + self.assertEqual(a.u(0), ('special', 0)) + self.assertEqual(a.u(2.5), ('general', 2.5)) + self.assertEqual(a.v(0), ('special', 0)) + self.assertEqual(a.v(2.5), ('general', 2.5)) + class CachedCostItem: _cost = 1 diff --git a/Misc/NEWS.d/next/Library/2025-11-01-14-44-09.gh-issue-140873.kfuc9B.rst b/Misc/NEWS.d/next/Library/2025-11-01-14-44-09.gh-issue-140873.kfuc9B.rst new file mode 100644 index 00000000000000..e15057640646d6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-01-14-44-09.gh-issue-140873.kfuc9B.rst @@ -0,0 +1,2 @@ +Add support of non-:term:`descriptor` callables in +:func:`functools.singledispatchmethod`. From 196f1519cd2d8134d7643536f13f2b2844bea65d Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Thu, 13 Nov 2025 17:58:47 +0000 Subject: [PATCH 035/638] gh-141004: Document `PyErr_RangedSyntaxLocationObject` (#141521) PyErr_RangedSyntaxLocationObject --- Doc/c-api/exceptions.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 0ee595a07acc77..d7fe9e2c9ec9b4 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -309,6 +309,14 @@ For convenience, some of these functions will always return a .. versionadded:: 3.4 +.. c:function:: void PyErr_RangedSyntaxLocationObject(PyObject *filename, int lineno, int col_offset, int end_lineno, int end_col_offset) + + Similar to :c:func:`PyErr_SyntaxLocationObject`, but also sets the + *end_lineno* and *end_col_offset* information for the current exception. + + .. versionadded:: 3.10 + + .. c:function:: void PyErr_SyntaxLocationEx(const char *filename, int lineno, int col_offset) Like :c:func:`PyErr_SyntaxLocationObject`, but *filename* is a byte string From 4fa80ce74c6d9f5159bdc5ec3596a194f0391e21 Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Fri, 14 Nov 2025 02:08:32 +0800 Subject: [PATCH 036/638] gh-139109: A new tracing JIT compiler frontend for CPython (GH-140310) This PR changes the current JIT model from trace projection to trace recording. Benchmarking: better pyperformance (about 1.7% overall) geomean versus current https://raw.githubusercontent.com/facebookexperimental/free-threading-benchmarking/refs/heads/main/results/bm-20251108-3.15.0a1%2B-7e2bc1d-JIT/bm-20251108-vultr-x86_64-Fidget%252dSpinner-tracing_jit-3.15.0a1%2B-7e2bc1d-vs-base.svg, 100% faster Richards on the most improved benchmark versus the current JIT. Slowdown of about 10-15% on the worst benchmark versus the current JIT. **Note: the fastest version isn't the one merged, as it relies on fixing bugs in the specializing interpreter, which is left to another PR**. The speedup in the merged version is about 1.1%. https://raw.githubusercontent.com/facebookexperimental/free-threading-benchmarking/refs/heads/main/results/bm-20251112-3.15.0a1%2B-f8a764a-JIT/bm-20251112-vultr-x86_64-Fidget%252dSpinner-tracing_jit-3.15.0a1%2B-f8a764a-vs-base.svg Stats: 50% more uops executed, 30% more traces entered the last time we ran them. It also suggests our trace lengths for a real trace recording JIT are too short, as a lot of trace too long aborts https://github.com/facebookexperimental/free-threading-benchmarking/blob/main/results/bm-20251023-3.15.0a1%2B-eb73378-CLANG%2CJIT/bm-20251023-vultr-x86_64-Fidget%252dSpinner-tracing_jit-3.15.0a1%2B-eb73378-pystats-vs-base.md . This new JIT frontend is already able to record/execute significantly more instructions than the previous JIT frontend. In this PR, we are now able to record through custom dunders, simple object creation, generators, etc. None of these were done by the old JIT frontend. Some custom dunders uops were discovered to be broken as part of this work gh-140277 The optimizer stack space check is disabled, as it's no longer valid to deal with underflow. Pros: * Ignoring the generated tracer code as it's automatically created, this is only additional 1k lines of code. The maintenance burden is handled by the DSL and code generator. * `optimizer.c` is now significantly simpler, as we don't have to do strange things to recover the bytecode from a trace. * The new JIT frontend is able to handle a lot more control-flow than the old one. * Tracing is very low overhead. We use the tail calling interpreter/computed goto interpreter to switch between tracing mode and non-tracing mode. I call this mechanism dual dispatch, as we have two dispatch tables dispatching to each other. Specialization is still enabled while tracing. * Better handling of polymorphism. We leverage the specializing interpreter for this. Cons: * (For now) requires tail calling interpreter or computed gotos. This means no Windows JIT for now :(. Not to fret, tail calling is coming soon to Windows though https://github.com/python/cpython/pull/139962 Design: * After each instruction, the `record_previous_inst` function/label is executed. This does as the name suggests. * The tracing interpreter lowers bytecode to uops directly so that it can obtain "fresh" values at the point of lowering. * The tracing version behaves nearly identical to the normal interpreter, in fact it even has specialization! This allows it to run without much of a slowdown when tracing. The actual cost of tracing is only a function call and writes to memory. * The tracing interpreter uses the specializing interpreter's deopt to naturally form the side exit chains. This allows it to side exit chain effectively, without repeating much code. We force a re-specializing when tracing a deopt. * The tracing interpreter can even handle goto errors/exceptions, but I chose to disable them for now as it's not tested. * Because we do not share interpreter dispatch, there is should be no significant slowdown to the original specializing interpreter on tailcall and computed got with JIT disabled. With JIT enabled, there might be a slowdown in the form of the JIT trying to trace. * Things that could have dynamic instruction pointer effects are guarded on. The guard deopts to a new instruction --- `_DYNAMIC_EXIT`. --- .github/workflows/jit.yml | 26 +- Include/cpython/pystats.h | 2 + Include/internal/pycore_backoff.h | 17 +- Include/internal/pycore_ceval.h | 2 + Include/internal/pycore_interp_structs.h | 4 +- Include/internal/pycore_opcode_metadata.h | 71 +- Include/internal/pycore_optimizer.h | 41 +- Include/internal/pycore_tstate.h | 39 +- Include/internal/pycore_uop.h | 12 +- Include/internal/pycore_uop_ids.h | 389 +++--- Include/internal/pycore_uop_metadata.h | 38 +- Lib/test/test_ast/test_ast.py | 4 +- Lib/test/test_capi/test_opt.py | 65 +- Lib/test/test_sys.py | 5 +- ...-10-18-21-50-44.gh-issue-139109.9QQOzN.rst | 1 + Modules/_testinternalcapi.c | 3 +- Objects/codeobject.c | 1 + Objects/frameobject.c | 6 +- Objects/funcobject.c | 6 +- Python/bytecodes.c | 192 ++- Python/ceval.c | 55 +- Python/ceval_macros.h | 67 +- Python/executor_cases.c.h | 135 ++- Python/generated_cases.c.h | 104 +- Python/instrumentation.c | 2 + Python/jit.c | 2 +- Python/opcode_targets.h | 526 +++++++- Python/optimizer.c | 1065 +++++++++-------- Python/optimizer_analysis.c | 54 +- Python/optimizer_bytecodes.c | 137 ++- Python/optimizer_cases.c.h | 153 ++- Python/optimizer_symbols.c | 44 +- Python/pystate.c | 27 +- Tools/c-analyzer/cpython/ignored.tsv | 1 + Tools/cases_generator/analyzer.py | 58 + Tools/cases_generator/generators_common.py | 17 +- .../opcode_metadata_generator.py | 4 +- Tools/cases_generator/target_generator.py | 26 +- Tools/cases_generator/tier2_generator.py | 54 +- .../cases_generator/uop_metadata_generator.py | 4 +- Tools/jit/template.c | 11 +- 41 files changed, 2407 insertions(+), 1063 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 62325250bd368e..3349eb042425dd 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -57,9 +57,10 @@ jobs: fail-fast: false matrix: target: - - i686-pc-windows-msvc/msvc - - x86_64-pc-windows-msvc/msvc - - aarch64-pc-windows-msvc/msvc +# To re-enable later when we support these. +# - i686-pc-windows-msvc/msvc +# - x86_64-pc-windows-msvc/msvc +# - aarch64-pc-windows-msvc/msvc - x86_64-apple-darwin/clang - aarch64-apple-darwin/clang - x86_64-unknown-linux-gnu/gcc @@ -70,15 +71,16 @@ jobs: llvm: - 21 include: - - target: i686-pc-windows-msvc/msvc - architecture: Win32 - runner: windows-2022 - - target: x86_64-pc-windows-msvc/msvc - architecture: x64 - runner: windows-2022 - - target: aarch64-pc-windows-msvc/msvc - architecture: ARM64 - runner: windows-11-arm +# To re-enable later when we support these. +# - target: i686-pc-windows-msvc/msvc +# architecture: Win32 +# runner: windows-2022 +# - target: x86_64-pc-windows-msvc/msvc +# architecture: x64 +# runner: windows-2022 +# - target: aarch64-pc-windows-msvc/msvc +# architecture: ARM64 +# runner: windows-11-arm - target: x86_64-apple-darwin/clang architecture: x86_64 runner: macos-15-intel diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index d0a925a3055485..1c94603c08b9b6 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -150,6 +150,8 @@ typedef struct _optimization_stats { uint64_t optimized_trace_length_hist[_Py_UOP_HIST_SIZE]; uint64_t optimizer_attempts; uint64_t optimizer_successes; + uint64_t optimizer_contradiction; + uint64_t optimizer_frame_overflow; uint64_t optimizer_failure_reason_no_memory; uint64_t remove_globals_builtins_changed; uint64_t remove_globals_incorrect_keys; diff --git a/Include/internal/pycore_backoff.h b/Include/internal/pycore_backoff.h index 454c8dde031ff4..71066f1bd9f19b 100644 --- a/Include/internal/pycore_backoff.h +++ b/Include/internal/pycore_backoff.h @@ -95,11 +95,24 @@ backoff_counter_triggers(_Py_BackoffCounter counter) return counter.value_and_backoff < UNREACHABLE_BACKOFF; } +static inline _Py_BackoffCounter +trigger_backoff_counter(void) +{ + _Py_BackoffCounter result; + result.value_and_backoff = 0; + return result; +} + // Initial JUMP_BACKWARD counter. // Must be larger than ADAPTIVE_COOLDOWN_VALUE, otherwise when JIT code is // invalidated we may construct a new trace before the bytecode has properly // re-specialized: -#define JUMP_BACKWARD_INITIAL_VALUE 4095 +// Note: this should be a prime number-1. This increases the likelihood of +// finding a "good" loop iteration to trace. +// For example, 4095 does not work for the nqueens benchmark on pyperformance +// as we always end up tracing the loop iteration's +// exhaustion iteration. Which aborts our current tracer. +#define JUMP_BACKWARD_INITIAL_VALUE 4000 #define JUMP_BACKWARD_INITIAL_BACKOFF 12 static inline _Py_BackoffCounter initial_jump_backoff_counter(void) @@ -112,7 +125,7 @@ initial_jump_backoff_counter(void) * Must be larger than ADAPTIVE_COOLDOWN_VALUE, * otherwise when a side exit warms up we may construct * a new trace before the Tier 1 code has properly re-specialized. */ -#define SIDE_EXIT_INITIAL_VALUE 4095 +#define SIDE_EXIT_INITIAL_VALUE 4000 #define SIDE_EXIT_INITIAL_BACKOFF 12 static inline _Py_BackoffCounter diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index fe72a0123ebea8..33b9fd053f70cb 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -392,6 +392,8 @@ _PyForIter_VirtualIteratorNext(PyThreadState* tstate, struct _PyInterpreterFrame #define SPECIAL___AEXIT__ 3 #define SPECIAL_MAX 3 +PyAPI_DATA(const _Py_CODEUNIT *) _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR; + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index e8cbe9d894e1c7..9e4504479cd9f0 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -14,8 +14,6 @@ extern "C" { #include "pycore_structs.h" // PyHamtObject #include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_typedefs.h" // _PyRuntimeState -#include "pycore_uop.h" // struct _PyUOpInstruction - #define CODE_MAX_WATCHERS 8 #define CONTEXT_MAX_WATCHERS 8 @@ -934,10 +932,10 @@ struct _is { PyObject *common_consts[NUM_COMMON_CONSTANTS]; bool jit; bool compiling; - struct _PyUOpInstruction *jit_uop_buffer; struct _PyExecutorObject *executor_list_head; struct _PyExecutorObject *executor_deletion_list_head; struct _PyExecutorObject *cold_executor; + struct _PyExecutorObject *cold_dynamic_executor; int executor_deletion_list_remaining_capacity; size_t executor_creation_counter; _rare_events rare_events; diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index bd6b84ec7fd908..548627dc7982ec 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1031,6 +1031,8 @@ enum InstructionFormat { #define HAS_ERROR_NO_POP_FLAG (4096) #define HAS_NO_SAVE_IP_FLAG (8192) #define HAS_PERIODIC_FLAG (16384) +#define HAS_UNPREDICTABLE_JUMP_FLAG (32768) +#define HAS_NEEDS_GUARD_IP_FLAG (65536) #define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG)) #define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG)) #define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG)) @@ -1046,6 +1048,8 @@ enum InstructionFormat { #define OPCODE_HAS_ERROR_NO_POP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ERROR_NO_POP_FLAG)) #define OPCODE_HAS_NO_SAVE_IP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NO_SAVE_IP_FLAG)) #define OPCODE_HAS_PERIODIC(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PERIODIC_FLAG)) +#define OPCODE_HAS_UNPREDICTABLE_JUMP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_UNPREDICTABLE_JUMP_FLAG)) +#define OPCODE_HAS_NEEDS_GUARD_IP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NEEDS_GUARD_IP_FLAG)) #define OPARG_SIMPLE 0 #define OPARG_CACHE_1 1 @@ -1062,7 +1066,7 @@ enum InstructionFormat { struct opcode_metadata { uint8_t valid_entry; uint8_t instr_format; - uint16_t flags; + uint32_t flags; }; extern const struct opcode_metadata _PyOpcode_opcode_metadata[267]; @@ -1077,7 +1081,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, [BINARY_OP_SUBSCR_DICT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG }, + [BINARY_OP_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [BINARY_OP_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_SUBSCR_LIST_SLICE] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, @@ -1094,22 +1098,22 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [BUILD_TEMPLATE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [CACHE] = { true, INSTR_FMT_IX, 0 }, - [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BOUND_METHOD_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_BOUND_METHOD_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [CALL_INTRINSIC_1] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_ISINSTANCE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW_BOUND_METHOD] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_KW_BOUND_METHOD] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [CALL_KW_NON_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [CALL_LEN] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1117,8 +1121,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_NON_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [CALL_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [CALL_STR_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_TUPLE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, @@ -1143,7 +1147,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [DELETE_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [END_FOR] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, [END_SEND] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_PURE_FLAG }, [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -1151,11 +1155,11 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [FORMAT_WITH_SPEC] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG }, + [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, + [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, + [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, + [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, [GET_AITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [GET_AWAITABLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1164,13 +1168,13 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_FROM] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -1183,8 +1187,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [IS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1197,7 +1201,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [LOAD_ATTR] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG }, @@ -1205,7 +1209,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1253,10 +1257,10 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [RESERVED] = { true, INSTR_FMT_IX, 0 }, [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [RESUME_CHECK] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [SETUP_ANNOTATIONS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -1292,7 +1296,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [WITH_EXCEPT_START] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [ANNOTATIONS_PLACEHOLDER] = { true, -1, HAS_PURE_FLAG }, [JUMP] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_IF_FALSE] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1406,6 +1410,9 @@ _PyOpcode_macro_expansion[256] = { [IMPORT_FROM] = { .nuops = 1, .uops = { { _IMPORT_FROM, OPARG_SIMPLE, 0 } } }, [IMPORT_NAME] = { .nuops = 1, .uops = { { _IMPORT_NAME, OPARG_SIMPLE, 0 } } }, [IS_OP] = { .nuops = 1, .uops = { { _IS_OP, OPARG_SIMPLE, 0 } } }, + [JUMP_BACKWARD] = { .nuops = 2, .uops = { { _CHECK_PERIODIC, OPARG_SIMPLE, 1 }, { _JUMP_BACKWARD_NO_INTERRUPT, OPARG_REPLACED, 1 } } }, + [JUMP_BACKWARD_NO_INTERRUPT] = { .nuops = 1, .uops = { { _JUMP_BACKWARD_NO_INTERRUPT, OPARG_REPLACED, 0 } } }, + [JUMP_BACKWARD_NO_JIT] = { .nuops = 2, .uops = { { _CHECK_PERIODIC, OPARG_SIMPLE, 1 }, { _JUMP_BACKWARD_NO_INTERRUPT, OPARG_REPLACED, 1 } } }, [LIST_APPEND] = { .nuops = 1, .uops = { { _LIST_APPEND, OPARG_SIMPLE, 0 } } }, [LIST_EXTEND] = { .nuops = 1, .uops = { { _LIST_EXTEND, OPARG_SIMPLE, 0 } } }, [LOAD_ATTR] = { .nuops = 1, .uops = { { _LOAD_ATTR, OPARG_SIMPLE, 8 } } }, diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 8ed5436eb6838c..653285a2c6b79b 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -21,14 +21,6 @@ typedef struct _PyExecutorLinkListNode { } _PyExecutorLinkListNode; -/* Bloom filter with m = 256 - * https://en.wikipedia.org/wiki/Bloom_filter */ -#define _Py_BLOOM_FILTER_WORDS 8 - -typedef struct { - uint32_t bits[_Py_BLOOM_FILTER_WORDS]; -} _PyBloomFilter; - typedef struct { uint8_t opcode; uint8_t oparg; @@ -44,7 +36,9 @@ typedef struct { typedef struct _PyExitData { uint32_t target; - uint16_t index; + uint16_t index:14; + uint16_t is_dynamic:1; + uint16_t is_control_flow:1; _Py_BackoffCounter temperature; struct _PyExecutorObject *executor; } _PyExitData; @@ -94,9 +88,8 @@ PyAPI_FUNC(void) _Py_Executors_InvalidateCold(PyInterpreterState *interp); // This value is arbitrary and was not optimized. #define JIT_CLEANUP_THRESHOLD 1000 -#define TRACE_STACK_SIZE 5 - -int _Py_uop_analyze_and_optimize(_PyInterpreterFrame *frame, +int _Py_uop_analyze_and_optimize( + PyFunctionObject *func, _PyUOpInstruction *trace, int trace_len, int curr_stackentries, _PyBloomFilter *dependencies); @@ -130,7 +123,7 @@ static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst) #define TY_ARENA_SIZE (UOP_MAX_TRACE_LENGTH * 5) // Need extras for root frame and for overflow frame (see TRACE_STACK_PUSH()) -#define MAX_ABSTRACT_FRAME_DEPTH (TRACE_STACK_SIZE + 2) +#define MAX_ABSTRACT_FRAME_DEPTH (16) // The maximum number of side exits that we can take before requiring forward // progress (and inserting a new ENTER_EXECUTOR instruction). In practice, this @@ -258,6 +251,7 @@ struct _Py_UOpsAbstractFrame { int stack_len; int locals_len; PyFunctionObject *func; + PyCodeObject *code; JitOptRef *stack_pointer; JitOptRef *stack; @@ -333,11 +327,11 @@ extern _Py_UOpsAbstractFrame *_Py_uop_frame_new( int curr_stackentries, JitOptRef *args, int arg_len); -extern int _Py_uop_frame_pop(JitOptContext *ctx); +extern int _Py_uop_frame_pop(JitOptContext *ctx, PyCodeObject *co, int curr_stackentries); PyAPI_FUNC(PyObject *) _Py_uop_symbols_test(PyObject *self, PyObject *ignored); -PyAPI_FUNC(int) _PyOptimizer_Optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *start, _PyExecutorObject **exec_ptr, int chain_depth); +PyAPI_FUNC(int) _PyOptimizer_Optimize(_PyInterpreterFrame *frame, PyThreadState *tstate); static inline _PyExecutorObject *_PyExecutor_FromExit(_PyExitData *exit) { @@ -346,6 +340,7 @@ static inline _PyExecutorObject *_PyExecutor_FromExit(_PyExitData *exit) } extern _PyExecutorObject *_PyExecutor_GetColdExecutor(void); +extern _PyExecutorObject *_PyExecutor_GetColdDynamicExecutor(void); PyAPI_FUNC(void) _PyExecutor_ClearExit(_PyExitData *exit); @@ -354,7 +349,9 @@ static inline int is_terminator(const _PyUOpInstruction *uop) int opcode = uop->opcode; return ( opcode == _EXIT_TRACE || - opcode == _JUMP_TO_TOP + opcode == _DEOPT || + opcode == _JUMP_TO_TOP || + opcode == _DYNAMIC_EXIT ); } @@ -365,6 +362,18 @@ PyAPI_FUNC(int) _PyDumpExecutors(FILE *out); extern void _Py_ClearExecutorDeletionList(PyInterpreterState *interp); #endif +int _PyJit_translate_single_bytecode_to_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, bool stop_tracing); + +int +_PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, + _Py_CODEUNIT *curr_instr, _Py_CODEUNIT *start_instr, + _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit, + int oparg); + +void _PyJit_FinalizeTracing(PyThreadState *tstate); + +void _PyJit_Tracer_InvalidateDependency(PyThreadState *old_tstate, void *obj); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index a44c523e2022a7..50048801b2e4ee 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -12,7 +12,8 @@ extern "C" { #include "pycore_freelist_state.h" // struct _Py_freelists #include "pycore_mimalloc.h" // struct _mimalloc_thread_state #include "pycore_qsbr.h" // struct qsbr - +#include "pycore_uop.h" // struct _PyUOpInstruction +#include "pycore_structs.h" #ifdef Py_GIL_DISABLED struct _gc_thread_state { @@ -21,6 +22,38 @@ struct _gc_thread_state { }; #endif +#if _Py_TIER2 +typedef struct _PyJitTracerInitialState { + int stack_depth; + int chain_depth; + struct _PyExitData *exit; + PyCodeObject *code; // Strong + PyFunctionObject *func; // Strong + _Py_CODEUNIT *start_instr; + _Py_CODEUNIT *close_loop_instr; + _Py_CODEUNIT *jump_backward_instr; +} _PyJitTracerInitialState; + +typedef struct _PyJitTracerPreviousState { + bool dependencies_still_valid; + bool instr_is_super; + int code_max_size; + int code_curr_size; + int instr_oparg; + int instr_stacklevel; + _Py_CODEUNIT *instr; + PyCodeObject *instr_code; // Strong + struct _PyInterpreterFrame *instr_frame; + _PyBloomFilter dependencies; +} _PyJitTracerPreviousState; + +typedef struct _PyJitTracerState { + _PyUOpInstruction *code_buffer; + _PyJitTracerInitialState initial_state; + _PyJitTracerPreviousState prev_state; +} _PyJitTracerState; +#endif + // Every PyThreadState is actually allocated as a _PyThreadStateImpl. The // PyThreadState fields are exposed as part of the C API, although most fields // are intended to be private. The _PyThreadStateImpl fields not exposed. @@ -85,7 +118,9 @@ typedef struct _PyThreadStateImpl { #if defined(Py_REF_DEBUG) && defined(Py_GIL_DISABLED) Py_ssize_t reftotal; // this thread's total refcount operations #endif - +#if _Py_TIER2 + _PyJitTracerState jit_tracer_state; +#endif } _PyThreadStateImpl; #ifdef __cplusplus diff --git a/Include/internal/pycore_uop.h b/Include/internal/pycore_uop.h index 4abefd3b95d21a..4e1b15af42caa3 100644 --- a/Include/internal/pycore_uop.h +++ b/Include/internal/pycore_uop.h @@ -35,10 +35,18 @@ typedef struct _PyUOpInstruction{ #endif } _PyUOpInstruction; -// This is the length of the trace we project initially. -#define UOP_MAX_TRACE_LENGTH 1200 +// This is the length of the trace we translate initially. +#define UOP_MAX_TRACE_LENGTH 3000 #define UOP_BUFFER_SIZE (UOP_MAX_TRACE_LENGTH * sizeof(_PyUOpInstruction)) +/* Bloom filter with m = 256 + * https://en.wikipedia.org/wiki/Bloom_filter */ +#define _Py_BLOOM_FILTER_WORDS 8 + +typedef struct { + uint32_t bits[_Py_BLOOM_FILTER_WORDS]; +} _PyBloomFilter; + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index ff1d75c0cb1938..7a33a5b84fd21a 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -81,101 +81,107 @@ extern "C" { #define _CHECK_STACK_SPACE 357 #define _CHECK_STACK_SPACE_OPERAND 358 #define _CHECK_VALIDITY 359 -#define _COLD_EXIT 360 -#define _COMPARE_OP 361 -#define _COMPARE_OP_FLOAT 362 -#define _COMPARE_OP_INT 363 -#define _COMPARE_OP_STR 364 -#define _CONTAINS_OP 365 -#define _CONTAINS_OP_DICT 366 -#define _CONTAINS_OP_SET 367 +#define _COLD_DYNAMIC_EXIT 360 +#define _COLD_EXIT 361 +#define _COMPARE_OP 362 +#define _COMPARE_OP_FLOAT 363 +#define _COMPARE_OP_INT 364 +#define _COMPARE_OP_STR 365 +#define _CONTAINS_OP 366 +#define _CONTAINS_OP_DICT 367 +#define _CONTAINS_OP_SET 368 #define _CONVERT_VALUE CONVERT_VALUE -#define _COPY 368 -#define _COPY_1 369 -#define _COPY_2 370 -#define _COPY_3 371 +#define _COPY 369 +#define _COPY_1 370 +#define _COPY_2 371 +#define _COPY_3 372 #define _COPY_FREE_VARS COPY_FREE_VARS -#define _CREATE_INIT_FRAME 372 +#define _CREATE_INIT_FRAME 373 #define _DELETE_ATTR DELETE_ATTR #define _DELETE_DEREF DELETE_DEREF #define _DELETE_FAST DELETE_FAST #define _DELETE_GLOBAL DELETE_GLOBAL #define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR -#define _DEOPT 373 +#define _DEOPT 374 #define _DICT_MERGE DICT_MERGE #define _DICT_UPDATE DICT_UPDATE -#define _DO_CALL 374 -#define _DO_CALL_FUNCTION_EX 375 -#define _DO_CALL_KW 376 +#define _DO_CALL 375 +#define _DO_CALL_FUNCTION_EX 376 +#define _DO_CALL_KW 377 +#define _DYNAMIC_EXIT 378 #define _END_FOR END_FOR #define _END_SEND END_SEND -#define _ERROR_POP_N 377 +#define _ERROR_POP_N 379 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _EXPAND_METHOD 378 -#define _EXPAND_METHOD_KW 379 -#define _FATAL_ERROR 380 +#define _EXPAND_METHOD 380 +#define _EXPAND_METHOD_KW 381 +#define _FATAL_ERROR 382 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 381 -#define _FOR_ITER_GEN_FRAME 382 -#define _FOR_ITER_TIER_TWO 383 +#define _FOR_ITER 383 +#define _FOR_ITER_GEN_FRAME 384 +#define _FOR_ITER_TIER_TWO 385 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BINARY_OP_EXTEND 384 -#define _GUARD_CALLABLE_ISINSTANCE 385 -#define _GUARD_CALLABLE_LEN 386 -#define _GUARD_CALLABLE_LIST_APPEND 387 -#define _GUARD_CALLABLE_STR_1 388 -#define _GUARD_CALLABLE_TUPLE_1 389 -#define _GUARD_CALLABLE_TYPE_1 390 -#define _GUARD_DORV_NO_DICT 391 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 392 -#define _GUARD_GLOBALS_VERSION 393 -#define _GUARD_IS_FALSE_POP 394 -#define _GUARD_IS_NONE_POP 395 -#define _GUARD_IS_NOT_NONE_POP 396 -#define _GUARD_IS_TRUE_POP 397 -#define _GUARD_KEYS_VERSION 398 -#define _GUARD_NOS_DICT 399 -#define _GUARD_NOS_FLOAT 400 -#define _GUARD_NOS_INT 401 -#define _GUARD_NOS_LIST 402 -#define _GUARD_NOS_NOT_NULL 403 -#define _GUARD_NOS_NULL 404 -#define _GUARD_NOS_OVERFLOWED 405 -#define _GUARD_NOS_TUPLE 406 -#define _GUARD_NOS_UNICODE 407 -#define _GUARD_NOT_EXHAUSTED_LIST 408 -#define _GUARD_NOT_EXHAUSTED_RANGE 409 -#define _GUARD_NOT_EXHAUSTED_TUPLE 410 -#define _GUARD_THIRD_NULL 411 -#define _GUARD_TOS_ANY_SET 412 -#define _GUARD_TOS_DICT 413 -#define _GUARD_TOS_FLOAT 414 -#define _GUARD_TOS_INT 415 -#define _GUARD_TOS_LIST 416 -#define _GUARD_TOS_OVERFLOWED 417 -#define _GUARD_TOS_SLICE 418 -#define _GUARD_TOS_TUPLE 419 -#define _GUARD_TOS_UNICODE 420 -#define _GUARD_TYPE_VERSION 421 -#define _GUARD_TYPE_VERSION_AND_LOCK 422 -#define _HANDLE_PENDING_AND_DEOPT 423 +#define _GUARD_BINARY_OP_EXTEND 386 +#define _GUARD_CALLABLE_ISINSTANCE 387 +#define _GUARD_CALLABLE_LEN 388 +#define _GUARD_CALLABLE_LIST_APPEND 389 +#define _GUARD_CALLABLE_STR_1 390 +#define _GUARD_CALLABLE_TUPLE_1 391 +#define _GUARD_CALLABLE_TYPE_1 392 +#define _GUARD_DORV_NO_DICT 393 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 394 +#define _GUARD_GLOBALS_VERSION 395 +#define _GUARD_IP_RETURN_GENERATOR 396 +#define _GUARD_IP_RETURN_VALUE 397 +#define _GUARD_IP_YIELD_VALUE 398 +#define _GUARD_IP__PUSH_FRAME 399 +#define _GUARD_IS_FALSE_POP 400 +#define _GUARD_IS_NONE_POP 401 +#define _GUARD_IS_NOT_NONE_POP 402 +#define _GUARD_IS_TRUE_POP 403 +#define _GUARD_KEYS_VERSION 404 +#define _GUARD_NOS_DICT 405 +#define _GUARD_NOS_FLOAT 406 +#define _GUARD_NOS_INT 407 +#define _GUARD_NOS_LIST 408 +#define _GUARD_NOS_NOT_NULL 409 +#define _GUARD_NOS_NULL 410 +#define _GUARD_NOS_OVERFLOWED 411 +#define _GUARD_NOS_TUPLE 412 +#define _GUARD_NOS_UNICODE 413 +#define _GUARD_NOT_EXHAUSTED_LIST 414 +#define _GUARD_NOT_EXHAUSTED_RANGE 415 +#define _GUARD_NOT_EXHAUSTED_TUPLE 416 +#define _GUARD_THIRD_NULL 417 +#define _GUARD_TOS_ANY_SET 418 +#define _GUARD_TOS_DICT 419 +#define _GUARD_TOS_FLOAT 420 +#define _GUARD_TOS_INT 421 +#define _GUARD_TOS_LIST 422 +#define _GUARD_TOS_OVERFLOWED 423 +#define _GUARD_TOS_SLICE 424 +#define _GUARD_TOS_TUPLE 425 +#define _GUARD_TOS_UNICODE 426 +#define _GUARD_TYPE_VERSION 427 +#define _GUARD_TYPE_VERSION_AND_LOCK 428 +#define _HANDLE_PENDING_AND_DEOPT 429 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 424 -#define _INIT_CALL_PY_EXACT_ARGS 425 -#define _INIT_CALL_PY_EXACT_ARGS_0 426 -#define _INIT_CALL_PY_EXACT_ARGS_1 427 -#define _INIT_CALL_PY_EXACT_ARGS_2 428 -#define _INIT_CALL_PY_EXACT_ARGS_3 429 -#define _INIT_CALL_PY_EXACT_ARGS_4 430 -#define _INSERT_NULL 431 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 430 +#define _INIT_CALL_PY_EXACT_ARGS 431 +#define _INIT_CALL_PY_EXACT_ARGS_0 432 +#define _INIT_CALL_PY_EXACT_ARGS_1 433 +#define _INIT_CALL_PY_EXACT_ARGS_2 434 +#define _INIT_CALL_PY_EXACT_ARGS_3 435 +#define _INIT_CALL_PY_EXACT_ARGS_4 436 +#define _INSERT_NULL 437 #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER #define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD @@ -185,177 +191,178 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _IS_NONE 432 +#define _IS_NONE 438 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 433 -#define _ITER_CHECK_RANGE 434 -#define _ITER_CHECK_TUPLE 435 -#define _ITER_JUMP_LIST 436 -#define _ITER_JUMP_RANGE 437 -#define _ITER_JUMP_TUPLE 438 -#define _ITER_NEXT_LIST 439 -#define _ITER_NEXT_LIST_TIER_TWO 440 -#define _ITER_NEXT_RANGE 441 -#define _ITER_NEXT_TUPLE 442 -#define _JUMP_TO_TOP 443 +#define _ITER_CHECK_LIST 439 +#define _ITER_CHECK_RANGE 440 +#define _ITER_CHECK_TUPLE 441 +#define _ITER_JUMP_LIST 442 +#define _ITER_JUMP_RANGE 443 +#define _ITER_JUMP_TUPLE 444 +#define _ITER_NEXT_LIST 445 +#define _ITER_NEXT_LIST_TIER_TWO 446 +#define _ITER_NEXT_RANGE 447 +#define _ITER_NEXT_TUPLE 448 +#define _JUMP_BACKWARD_NO_INTERRUPT JUMP_BACKWARD_NO_INTERRUPT +#define _JUMP_TO_TOP 449 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 444 -#define _LOAD_ATTR_CLASS 445 +#define _LOAD_ATTR 450 +#define _LOAD_ATTR_CLASS 451 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 446 -#define _LOAD_ATTR_METHOD_LAZY_DICT 447 -#define _LOAD_ATTR_METHOD_NO_DICT 448 -#define _LOAD_ATTR_METHOD_WITH_VALUES 449 -#define _LOAD_ATTR_MODULE 450 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 451 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 452 -#define _LOAD_ATTR_PROPERTY_FRAME 453 -#define _LOAD_ATTR_SLOT 454 -#define _LOAD_ATTR_WITH_HINT 455 +#define _LOAD_ATTR_INSTANCE_VALUE 452 +#define _LOAD_ATTR_METHOD_LAZY_DICT 453 +#define _LOAD_ATTR_METHOD_NO_DICT 454 +#define _LOAD_ATTR_METHOD_WITH_VALUES 455 +#define _LOAD_ATTR_MODULE 456 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 457 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 458 +#define _LOAD_ATTR_PROPERTY_FRAME 459 +#define _LOAD_ATTR_SLOT 460 +#define _LOAD_ATTR_WITH_HINT 461 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 456 +#define _LOAD_BYTECODE 462 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 457 -#define _LOAD_CONST_INLINE_BORROW 458 -#define _LOAD_CONST_UNDER_INLINE 459 -#define _LOAD_CONST_UNDER_INLINE_BORROW 460 +#define _LOAD_CONST_INLINE 463 +#define _LOAD_CONST_INLINE_BORROW 464 +#define _LOAD_CONST_UNDER_INLINE 465 +#define _LOAD_CONST_UNDER_INLINE_BORROW 466 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 461 -#define _LOAD_FAST_0 462 -#define _LOAD_FAST_1 463 -#define _LOAD_FAST_2 464 -#define _LOAD_FAST_3 465 -#define _LOAD_FAST_4 466 -#define _LOAD_FAST_5 467 -#define _LOAD_FAST_6 468 -#define _LOAD_FAST_7 469 +#define _LOAD_FAST 467 +#define _LOAD_FAST_0 468 +#define _LOAD_FAST_1 469 +#define _LOAD_FAST_2 470 +#define _LOAD_FAST_3 471 +#define _LOAD_FAST_4 472 +#define _LOAD_FAST_5 473 +#define _LOAD_FAST_6 474 +#define _LOAD_FAST_7 475 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_BORROW 470 -#define _LOAD_FAST_BORROW_0 471 -#define _LOAD_FAST_BORROW_1 472 -#define _LOAD_FAST_BORROW_2 473 -#define _LOAD_FAST_BORROW_3 474 -#define _LOAD_FAST_BORROW_4 475 -#define _LOAD_FAST_BORROW_5 476 -#define _LOAD_FAST_BORROW_6 477 -#define _LOAD_FAST_BORROW_7 478 +#define _LOAD_FAST_BORROW 476 +#define _LOAD_FAST_BORROW_0 477 +#define _LOAD_FAST_BORROW_1 478 +#define _LOAD_FAST_BORROW_2 479 +#define _LOAD_FAST_BORROW_3 480 +#define _LOAD_FAST_BORROW_4 481 +#define _LOAD_FAST_BORROW_5 482 +#define _LOAD_FAST_BORROW_6 483 +#define _LOAD_FAST_BORROW_7 484 #define _LOAD_FAST_BORROW_LOAD_FAST_BORROW LOAD_FAST_BORROW_LOAD_FAST_BORROW #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 479 -#define _LOAD_GLOBAL_BUILTINS 480 -#define _LOAD_GLOBAL_MODULE 481 +#define _LOAD_GLOBAL 485 +#define _LOAD_GLOBAL_BUILTINS 486 +#define _LOAD_GLOBAL_MODULE 487 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 482 -#define _LOAD_SMALL_INT_0 483 -#define _LOAD_SMALL_INT_1 484 -#define _LOAD_SMALL_INT_2 485 -#define _LOAD_SMALL_INT_3 486 -#define _LOAD_SPECIAL 487 +#define _LOAD_SMALL_INT 488 +#define _LOAD_SMALL_INT_0 489 +#define _LOAD_SMALL_INT_1 490 +#define _LOAD_SMALL_INT_2 491 +#define _LOAD_SMALL_INT_3 492 +#define _LOAD_SPECIAL 493 #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 488 +#define _MAKE_CALLARGS_A_TUPLE 494 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 489 +#define _MAKE_WARM 495 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 490 -#define _MAYBE_EXPAND_METHOD_KW 491 -#define _MONITOR_CALL 492 -#define _MONITOR_CALL_KW 493 -#define _MONITOR_JUMP_BACKWARD 494 -#define _MONITOR_RESUME 495 +#define _MAYBE_EXPAND_METHOD 496 +#define _MAYBE_EXPAND_METHOD_KW 497 +#define _MONITOR_CALL 498 +#define _MONITOR_CALL_KW 499 +#define _MONITOR_JUMP_BACKWARD 500 +#define _MONITOR_RESUME 501 #define _NOP NOP -#define _POP_CALL 496 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW 497 -#define _POP_CALL_ONE 498 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 499 -#define _POP_CALL_TWO 500 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 501 +#define _POP_CALL 502 +#define _POP_CALL_LOAD_CONST_INLINE_BORROW 503 +#define _POP_CALL_ONE 504 +#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 505 +#define _POP_CALL_TWO 506 +#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 507 #define _POP_EXCEPT POP_EXCEPT #define _POP_ITER POP_ITER -#define _POP_JUMP_IF_FALSE 502 -#define _POP_JUMP_IF_TRUE 503 +#define _POP_JUMP_IF_FALSE 508 +#define _POP_JUMP_IF_TRUE 509 #define _POP_TOP POP_TOP -#define _POP_TOP_FLOAT 504 -#define _POP_TOP_INT 505 -#define _POP_TOP_LOAD_CONST_INLINE 506 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 507 -#define _POP_TOP_NOP 508 -#define _POP_TOP_UNICODE 509 -#define _POP_TWO 510 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW 511 +#define _POP_TOP_FLOAT 510 +#define _POP_TOP_INT 511 +#define _POP_TOP_LOAD_CONST_INLINE 512 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 513 +#define _POP_TOP_NOP 514 +#define _POP_TOP_UNICODE 515 +#define _POP_TWO 516 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW 517 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 512 +#define _PUSH_FRAME 518 #define _PUSH_NULL PUSH_NULL -#define _PUSH_NULL_CONDITIONAL 513 -#define _PY_FRAME_GENERAL 514 -#define _PY_FRAME_KW 515 -#define _QUICKEN_RESUME 516 -#define _REPLACE_WITH_TRUE 517 +#define _PUSH_NULL_CONDITIONAL 519 +#define _PY_FRAME_GENERAL 520 +#define _PY_FRAME_KW 521 +#define _QUICKEN_RESUME 522 +#define _REPLACE_WITH_TRUE 523 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 518 -#define _SEND 519 -#define _SEND_GEN_FRAME 520 +#define _SAVE_RETURN_OFFSET 524 +#define _SEND 525 +#define _SEND_GEN_FRAME 526 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 521 -#define _STORE_ATTR 522 -#define _STORE_ATTR_INSTANCE_VALUE 523 -#define _STORE_ATTR_SLOT 524 -#define _STORE_ATTR_WITH_HINT 525 +#define _START_EXECUTOR 527 +#define _STORE_ATTR 528 +#define _STORE_ATTR_INSTANCE_VALUE 529 +#define _STORE_ATTR_SLOT 530 +#define _STORE_ATTR_WITH_HINT 531 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 526 -#define _STORE_FAST_0 527 -#define _STORE_FAST_1 528 -#define _STORE_FAST_2 529 -#define _STORE_FAST_3 530 -#define _STORE_FAST_4 531 -#define _STORE_FAST_5 532 -#define _STORE_FAST_6 533 -#define _STORE_FAST_7 534 +#define _STORE_FAST 532 +#define _STORE_FAST_0 533 +#define _STORE_FAST_1 534 +#define _STORE_FAST_2 535 +#define _STORE_FAST_3 536 +#define _STORE_FAST_4 537 +#define _STORE_FAST_5 538 +#define _STORE_FAST_6 539 +#define _STORE_FAST_7 540 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 535 -#define _STORE_SUBSCR 536 -#define _STORE_SUBSCR_DICT 537 -#define _STORE_SUBSCR_LIST_INT 538 -#define _SWAP 539 -#define _SWAP_2 540 -#define _SWAP_3 541 -#define _TIER2_RESUME_CHECK 542 -#define _TO_BOOL 543 +#define _STORE_SLICE 541 +#define _STORE_SUBSCR 542 +#define _STORE_SUBSCR_DICT 543 +#define _STORE_SUBSCR_LIST_INT 544 +#define _SWAP 545 +#define _SWAP_2 546 +#define _SWAP_3 547 +#define _TIER2_RESUME_CHECK 548 +#define _TO_BOOL 549 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT -#define _TO_BOOL_LIST 544 +#define _TO_BOOL_LIST 550 #define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR 545 +#define _TO_BOOL_STR 551 #define _UNARY_INVERT UNARY_INVERT #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 546 -#define _UNPACK_SEQUENCE_LIST 547 -#define _UNPACK_SEQUENCE_TUPLE 548 -#define _UNPACK_SEQUENCE_TWO_TUPLE 549 +#define _UNPACK_SEQUENCE 552 +#define _UNPACK_SEQUENCE_LIST 553 +#define _UNPACK_SEQUENCE_TUPLE 554 +#define _UNPACK_SEQUENCE_TWO_TUPLE 555 #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 549 +#define MAX_UOP_ID 555 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 1248771996943b..d5a3c362d875e6 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -11,7 +11,7 @@ extern "C" { #include #include "pycore_uop_ids.h" -extern const uint16_t _PyUop_Flags[MAX_UOP_ID+1]; +extern const uint32_t _PyUop_Flags[MAX_UOP_ID+1]; typedef struct _rep_range { uint8_t start; uint8_t stop; } ReplicationRange; extern const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1]; extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1]; @@ -19,7 +19,7 @@ extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1]; extern int _PyUop_num_popped(int opcode, int oparg); #ifdef NEED_OPCODE_METADATA -const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { +const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_NOP] = HAS_PURE_FLAG, [_CHECK_PERIODIC] = HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CHECK_PERIODIC_IF_NOT_YIELD_FROM] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -128,12 +128,12 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_DELETE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_INTRINSIC_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_INTRINSIC_2] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_RETURN_VALUE] = HAS_ESCAPES_FLAG, + [_RETURN_VALUE] = HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_GET_AITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GET_AWAITABLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SEND_GEN_FRAME] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_YIELD_VALUE] = HAS_ARG_FLAG, + [_YIELD_VALUE] = HAS_ARG_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_POP_EXCEPT] = HAS_ESCAPES_FLAG, [_LOAD_COMMON_CONSTANT] = HAS_ARG_FLAG, [_LOAD_BUILD_CLASS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -256,7 +256,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_INIT_CALL_PY_EXACT_ARGS_3] = HAS_PURE_FLAG, [_INIT_CALL_PY_EXACT_ARGS_4] = HAS_PURE_FLAG, [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG | HAS_PURE_FLAG, - [_PUSH_FRAME] = 0, + [_PUSH_FRAME] = HAS_NEEDS_GUARD_IP_FLAG, [_GUARD_NOS_NULL] = HAS_DEOPT_FLAG, [_GUARD_NOS_NOT_NULL] = HAS_EXIT_FLAG, [_GUARD_THIRD_NULL] = HAS_DEOPT_FLAG, @@ -293,7 +293,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_MAKE_CALLARGS_A_TUPLE] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SET_FUNCTION_ATTRIBUTE] = HAS_ARG_FLAG, - [_RETURN_GENERATOR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_RETURN_GENERATOR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_BUILD_SLICE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CONVERT_VALUE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_FORMAT_SIMPLE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -315,6 +315,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CHECK_STACK_SPACE_OPERAND] = HAS_DEOPT_FLAG, [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG, [_EXIT_TRACE] = HAS_ESCAPES_FLAG, + [_DYNAMIC_EXIT] = HAS_ESCAPES_FLAG, [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, [_LOAD_CONST_INLINE] = HAS_PURE_FLAG, [_POP_TOP_LOAD_CONST_INLINE] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, @@ -336,7 +337,12 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_HANDLE_PENDING_AND_DEOPT] = HAS_ESCAPES_FLAG, [_ERROR_POP_N] = HAS_ARG_FLAG, [_TIER2_RESUME_CHECK] = HAS_PERIODIC_FLAG, - [_COLD_EXIT] = HAS_ESCAPES_FLAG, + [_COLD_EXIT] = 0, + [_COLD_DYNAMIC_EXIT] = 0, + [_GUARD_IP__PUSH_FRAME] = HAS_EXIT_FLAG, + [_GUARD_IP_YIELD_VALUE] = HAS_EXIT_FLAG, + [_GUARD_IP_RETURN_VALUE] = HAS_EXIT_FLAG, + [_GUARD_IP_RETURN_GENERATOR] = HAS_EXIT_FLAG, }; const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1] = { @@ -419,6 +425,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE", [_CHECK_STACK_SPACE_OPERAND] = "_CHECK_STACK_SPACE_OPERAND", [_CHECK_VALIDITY] = "_CHECK_VALIDITY", + [_COLD_DYNAMIC_EXIT] = "_COLD_DYNAMIC_EXIT", [_COLD_EXIT] = "_COLD_EXIT", [_COMPARE_OP] = "_COMPARE_OP", [_COMPARE_OP_FLOAT] = "_COMPARE_OP_FLOAT", @@ -443,6 +450,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_DEOPT] = "_DEOPT", [_DICT_MERGE] = "_DICT_MERGE", [_DICT_UPDATE] = "_DICT_UPDATE", + [_DYNAMIC_EXIT] = "_DYNAMIC_EXIT", [_END_FOR] = "_END_FOR", [_END_SEND] = "_END_SEND", [_ERROR_POP_N] = "_ERROR_POP_N", @@ -471,6 +479,10 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_GUARD_DORV_NO_DICT] = "_GUARD_DORV_NO_DICT", [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT", [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION", + [_GUARD_IP_RETURN_GENERATOR] = "_GUARD_IP_RETURN_GENERATOR", + [_GUARD_IP_RETURN_VALUE] = "_GUARD_IP_RETURN_VALUE", + [_GUARD_IP_YIELD_VALUE] = "_GUARD_IP_YIELD_VALUE", + [_GUARD_IP__PUSH_FRAME] = "_GUARD_IP__PUSH_FRAME", [_GUARD_IS_FALSE_POP] = "_GUARD_IS_FALSE_POP", [_GUARD_IS_NONE_POP] = "_GUARD_IS_NONE_POP", [_GUARD_IS_NOT_NONE_POP] = "_GUARD_IS_NOT_NONE_POP", @@ -1261,6 +1273,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _EXIT_TRACE: return 0; + case _DYNAMIC_EXIT: + return 0; case _CHECK_VALIDITY: return 0; case _LOAD_CONST_INLINE: @@ -1305,6 +1319,16 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _COLD_EXIT: return 0; + case _COLD_DYNAMIC_EXIT: + return 0; + case _GUARD_IP__PUSH_FRAME: + return 0; + case _GUARD_IP_YIELD_VALUE: + return 0; + case _GUARD_IP_RETURN_VALUE: + return 0; + case _GUARD_IP_RETURN_GENERATOR: + return 0; default: return -1; } diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index fb4a441ca64772..608ffdfad1209a 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -3057,8 +3057,8 @@ def test_source_segment_missing_info(self): class NodeTransformerTests(ASTTestMixin, unittest.TestCase): def assertASTTransformation(self, transformer_class, - initial_code, expected_code): - initial_ast = ast.parse(dedent(initial_code)) + code, expected_code): + initial_ast = ast.parse(dedent(code)) expected_ast = ast.parse(dedent(expected_code)) transformer = transformer_class() diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index e65556fb28f92d..f06c6cbda2976c 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -422,32 +422,6 @@ def testfunc(n, m): uops = get_opnames(ex) self.assertIn("_FOR_ITER_TIER_TWO", uops) - def test_confidence_score(self): - def testfunc(n): - bits = 0 - for i in range(n): - if i & 0x01: - bits += 1 - if i & 0x02: - bits += 1 - if i&0x04: - bits += 1 - if i&0x08: - bits += 1 - if i&0x10: - bits += 1 - return bits - - x = testfunc(TIER2_THRESHOLD * 2) - - self.assertEqual(x, TIER2_THRESHOLD * 5) - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - ops = list(iter_opnames(ex)) - #Since branch is 50/50 the trace could go either way. - count = ops.count("_GUARD_IS_TRUE_POP") + ops.count("_GUARD_IS_FALSE_POP") - self.assertLessEqual(count, 2) - @requires_specialization @unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds") @@ -847,38 +821,7 @@ def testfunc(n): self.assertLessEqual(len(guard_nos_unicode_count), 1) self.assertIn("_COMPARE_OP_STR", uops) - def test_type_inconsistency(self): - ns = {} - src = textwrap.dedent(""" - def testfunc(n): - for i in range(n): - x = _test_global + _test_global - """) - exec(src, ns, ns) - testfunc = ns['testfunc'] - ns['_test_global'] = 0 - _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) - self.assertIsNone(ex) - ns['_test_global'] = 1 - _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertNotIn("_GUARD_TOS_INT", uops) - self.assertNotIn("_GUARD_NOS_INT", uops) - self.assertNotIn("_BINARY_OP_ADD_INT", uops) - self.assertNotIn("_POP_TWO_LOAD_CONST_INLINE_BORROW", uops) - # Try again, but between the runs, set the global to a float. - # This should result in no executor the second time. - ns = {} - exec(src, ns, ns) - testfunc = ns['testfunc'] - ns['_test_global'] = 0 - _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) - self.assertIsNone(ex) - ns['_test_global'] = 3.14 - _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) - self.assertIsNone(ex) - + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_checks_sequential(self): def dummy12(x): return x - 1 @@ -907,6 +850,7 @@ def testfunc(n): largest_stack = _testinternalcapi.get_co_framesize(dummy13.__code__) self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_checks_nested(self): def dummy12(x): return x + 3 @@ -937,6 +881,7 @@ def testfunc(n): ) self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_checks_several_calls(self): def dummy12(x): return x + 3 @@ -972,6 +917,7 @@ def testfunc(n): ) self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_checks_several_calls_different_order(self): # same as `several_calls` but with top-level calls reversed def dummy12(x): @@ -1008,6 +954,7 @@ def testfunc(n): ) self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_complex(self): def dummy0(x): return x @@ -1057,6 +1004,7 @@ def testfunc(n): ("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands ) + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_checks_large_framesize(self): # Create a function with a large framesize. This ensures _CHECK_STACK_SPACE is # actually doing its job. Note that the resulting trace hits @@ -1118,6 +1066,7 @@ def testfunc(n): ("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands ) + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_checks_recursion(self): def dummy15(x): while x > 0: diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 9d3248d972e8d1..798f58737b1bf6 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -2253,9 +2253,10 @@ def frame_2_jit(expected: bool) -> None: def frame_3_jit() -> None: # JITs just before the last loop: - for i in range(_testinternalcapi.TIER2_THRESHOLD + 1): + # 1 extra iteration for tracing. + for i in range(_testinternalcapi.TIER2_THRESHOLD + 2): # Careful, doing this in the reverse order breaks tracing: - expected = {enabled} and i == _testinternalcapi.TIER2_THRESHOLD + expected = {enabled} and i >= _testinternalcapi.TIER2_THRESHOLD + 1 assert sys._jit.is_active() is expected frame_2_jit(expected) assert sys._jit.is_active() is expected diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst new file mode 100644 index 00000000000000..40b9d19ee42968 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst @@ -0,0 +1 @@ +A new tracing frontend for the JIT compiler has been implemented. Patch by Ken Jin. Design for CPython by Ken Jin, Mark Shannon and Brandt Bucher. diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 6514ca7f3cd6de..89e558b0fe8933 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -2661,7 +2661,8 @@ module_exec(PyObject *module) } if (PyModule_Add(module, "TIER2_THRESHOLD", - PyLong_FromLong(JUMP_BACKWARD_INITIAL_VALUE + 1)) < 0) { + // + 1 more due to one loop spent on tracing. + PyLong_FromLong(JUMP_BACKWARD_INITIAL_VALUE + 2)) < 0) { return 1; } diff --git a/Objects/codeobject.c b/Objects/codeobject.c index fc3f5d9dde0bc1..3aea2038fd17e7 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -2432,6 +2432,7 @@ code_dealloc(PyObject *self) PyMem_Free(co_extra); } #ifdef _Py_TIER2 + _PyJit_Tracer_InvalidateDependency(tstate, self); if (co->co_executors != NULL) { clear_executors(co); } diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 0cae3703d1d0c6..b652973600c17d 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -17,6 +17,7 @@ #include "frameobject.h" // PyFrameLocalsProxyObject #include "opcode.h" // EXTENDED_ARG +#include "pycore_optimizer.h" #include "clinic/frameobject.c.h" @@ -260,7 +261,10 @@ framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) return -1; } - _Py_Executors_InvalidateDependency(PyInterpreterState_Get(), co, 1); +#if _Py_TIER2 + _Py_Executors_InvalidateDependency(_PyInterpreterState_GET(), co, 1); + _PyJit_Tracer_InvalidateDependency(_PyThreadState_GET(), co); +#endif _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); _PyStackRef oldvalue = fast[i]; diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 43198aaf8a7048..b659ac8023373b 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -11,7 +11,7 @@ #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_stats.h" #include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() - +#include "pycore_optimizer.h" // _PyJit_Tracer_InvalidateDependency static const char * func_event_name(PyFunction_WatchEvent event) { @@ -1151,6 +1151,10 @@ func_dealloc(PyObject *self) if (_PyObject_ResurrectEnd(self)) { return; } +#if _Py_TIER2 + _Py_Executors_InvalidateDependency(_PyInterpreterState_GET(), self, 1); + _PyJit_Tracer_InvalidateDependency(_PyThreadState_GET(), self); +#endif _PyObject_GC_UNTRACK(op); FT_CLEAR_WEAKREFS(self, op->func_weakreflist); (void)func_clear((PyObject*)op); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 6ebd9ebdfce1bb..2c798855a71f55 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2938,8 +2938,8 @@ dummy_func( JUMP_BACKWARD_JIT, }; - tier1 op(_SPECIALIZE_JUMP_BACKWARD, (--)) { - #if ENABLE_SPECIALIZATION_FT + specializing tier1 op(_SPECIALIZE_JUMP_BACKWARD, (--)) { + #if ENABLE_SPECIALIZATION if (this_instr->op.code == JUMP_BACKWARD) { uint8_t desired = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, desired); @@ -2953,25 +2953,21 @@ dummy_func( tier1 op(_JIT, (--)) { #ifdef _Py_TIER2 _Py_BackoffCounter counter = this_instr[1].counter; - if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT) { - _Py_CODEUNIT *start = this_instr; - /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ + if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && + this_instr->op.code == JUMP_BACKWARD_JIT && + next_instr->op.code != ENTER_EXECUTOR) { + /* Back up over EXTENDED_ARGs so executor is inserted at the correct place */ + _Py_CODEUNIT *insert_exec_at = this_instr; while (oparg > 255) { oparg >>= 8; - start--; + insert_exec_at--; } - _PyExecutorObject *executor; - int optimized = _PyOptimizer_Optimize(frame, start, &executor, 0); - if (optimized <= 0) { - this_instr[1].counter = restart_backoff_counter(counter); - ERROR_IF(optimized < 0); + int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, oparg); + if (succ) { + ENTER_TRACING(); } else { - this_instr[1].counter = initial_jump_backoff_counter(); - assert(tstate->current_executor == NULL); - assert(executor != tstate->interp->cold_executor); - tstate->jit_exit = NULL; - TIER1_TO_TIER2(executor); + this_instr[1].counter = restart_backoff_counter(counter); } } else { @@ -3017,6 +3013,10 @@ dummy_func( tier1 inst(ENTER_EXECUTOR, (--)) { #ifdef _Py_TIER2 + if (IS_JIT_TRACING()) { + next_instr = this_instr; + goto stop_tracing; + } PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; assert(executor->vm_data.index == INSTR_OFFSET() - 1); @@ -3078,7 +3078,7 @@ dummy_func( macro(POP_JUMP_IF_NOT_NONE) = unused/1 + _IS_NONE + _POP_JUMP_IF_FALSE; - tier1 inst(JUMP_BACKWARD_NO_INTERRUPT, (--)) { + replaced inst(JUMP_BACKWARD_NO_INTERRUPT, (--)) { /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. @@ -5245,21 +5245,42 @@ dummy_func( tier2 op(_EXIT_TRACE, (exit_p/4 --)) { _PyExitData *exit = (_PyExitData *)exit_p; #if defined(Py_DEBUG) && !defined(_Py_JIT) - _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; + const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER) + ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame)) + + exit->target; OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - if (frame->lltrace >= 2) { + if (frame->lltrace >= 3) { printf("SIDE EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); - printf(", exit %tu, temp %d, target %d -> %s]\n", + printf(", exit %tu, temp %d, target %d -> %s, is_control_flow %d]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyFrame_GetBytecode(frame)), - _PyOpcode_OpName[target->op.code]); + _PyOpcode_OpName[target->op.code], exit->is_control_flow); } #endif tstate->jit_exit = exit; TIER2_TO_TIER2(exit->executor); } + tier2 op(_DYNAMIC_EXIT, (exit_p/4 --)) { + #if defined(Py_DEBUG) && !defined(_Py_JIT) + _PyExitData *exit = (_PyExitData *)exit_p; + _Py_CODEUNIT *target = frame->instr_ptr; + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + if (frame->lltrace >= 3) { + printf("DYNAMIC EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(", exit %tu, temp %d, target %d -> %s]\n", + exit - current_executor->exits, exit->temperature.value_and_backoff, + (int)(target - _PyFrame_GetBytecode(frame)), + _PyOpcode_OpName[target->op.code]); + } + #endif + // Disabled for now (gh-139109) as it slows down dynamic code tremendously. + // Compile and jump to the cold dynamic executors in the future. + GOTO_TIER_ONE(frame->instr_ptr); + } + tier2 op(_CHECK_VALIDITY, (--)) { DEOPT_IF(!current_executor->vm_data.valid); } @@ -5369,7 +5390,8 @@ dummy_func( } tier2 op(_DEOPT, (--)) { - GOTO_TIER_ONE(_PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + GOTO_TIER_ONE((frame->owner == FRAME_OWNED_BY_INTERPRETER) + ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); } tier2 op(_HANDLE_PENDING_AND_DEOPT, (--)) { @@ -5399,32 +5421,76 @@ dummy_func( tier2 op(_COLD_EXIT, ( -- )) { _PyExitData *exit = tstate->jit_exit; assert(exit != NULL); + assert(frame->owner < FRAME_OWNED_BY_INTERPRETER); _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; _Py_BackoffCounter temperature = exit->temperature; - if (!backoff_counter_triggers(temperature)) { - exit->temperature = advance_backoff_counter(temperature); - GOTO_TIER_ONE(target); - } _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { PyCodeObject *code = _PyFrame_GetCode(frame); executor = code->co_executors->executors[target->op.arg]; Py_INCREF(executor); + assert(tstate->jit_exit == exit); + exit->executor = executor; + TIER2_TO_TIER2(exit->executor); } else { + if (!backoff_counter_triggers(temperature)) { + exit->temperature = advance_backoff_counter(temperature); + GOTO_TIER_ONE(target); + } _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); assert(tstate->current_executor == (PyObject *)previous_executor); - int chain_depth = previous_executor->vm_data.chain_depth + 1; - int optimized = _PyOptimizer_Optimize(frame, target, &executor, chain_depth); - if (optimized <= 0) { - exit->temperature = restart_backoff_counter(temperature); - GOTO_TIER_ONE(optimized < 0 ? NULL : target); + // For control-flow guards, we don't want to increase the chain depth, as those don't actually + // represent deopts but rather just normal programs! + int chain_depth = previous_executor->vm_data.chain_depth + !exit->is_control_flow; + // Note: it's safe to use target->op.arg here instead of the oparg given by EXTENDED_ARG. + // The invariant in the optimizer is the deopt target always points back to the first EXTENDED_ARG. + // So setting it to anything else is wrong. + int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, target->op.arg); + exit->temperature = restart_backoff_counter(exit->temperature); + if (succ) { + GOTO_TIER_ONE_CONTINUE_TRACING(target); } - exit->temperature = initial_temperature_backoff_counter(); + GOTO_TIER_ONE(target); + } + } + + tier2 op(_COLD_DYNAMIC_EXIT, ( -- )) { + // TODO (gh-139109): This should be similar to _COLD_EXIT in the future. + _Py_CODEUNIT *target = frame->instr_ptr; + GOTO_TIER_ONE(target); + } + + tier2 op(_GUARD_IP__PUSH_FRAME, (ip/4 --)) { + _Py_CODEUNIT *target = frame->instr_ptr + IP_OFFSET_OF(_PUSH_FRAME); + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += IP_OFFSET_OF(_PUSH_FRAME); + EXIT_IF(true); + } + } + + tier2 op(_GUARD_IP_YIELD_VALUE, (ip/4 --)) { + _Py_CODEUNIT *target = frame->instr_ptr + IP_OFFSET_OF(YIELD_VALUE); + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += IP_OFFSET_OF(YIELD_VALUE); + EXIT_IF(true); + } + } + + tier2 op(_GUARD_IP_RETURN_VALUE, (ip/4 --)) { + _Py_CODEUNIT *target = frame->instr_ptr + IP_OFFSET_OF(RETURN_VALUE); + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += IP_OFFSET_OF(RETURN_VALUE); + EXIT_IF(true); + } + } + + tier2 op(_GUARD_IP_RETURN_GENERATOR, (ip/4 --)) { + _Py_CODEUNIT *target = frame->instr_ptr + IP_OFFSET_OF(RETURN_GENERATOR); + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += IP_OFFSET_OF(RETURN_GENERATOR); + EXIT_IF(true); } - assert(tstate->jit_exit == exit); - exit->executor = executor; - TIER2_TO_TIER2(exit->executor); } label(pop_2_error) { @@ -5571,6 +5637,62 @@ dummy_func( DISPATCH(); } + label(record_previous_inst) { +#if _Py_TIER2 + assert(IS_JIT_TRACING()); + int opcode = next_instr->op.code; + bool stop_tracing = (opcode == WITH_EXCEPT_START || + opcode == RERAISE || opcode == CLEANUP_THROW || + opcode == PUSH_EXC_INFO || opcode == INTERPRETER_EXIT); + int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing); + if (full) { + LEAVE_TRACING(); + int err = stop_tracing_and_jit(tstate, frame); + ERROR_IF(err < 0); + DISPATCH_GOTO_NON_TRACING(); + } + // Super instructions. Instruction deopted. There's a mismatch in what the stack expects + // in the optimizer. So we have to reflect in the trace correctly. + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + if ((_tstate->jit_tracer_state.prev_state.instr->op.code == CALL_LIST_APPEND && + opcode == POP_TOP) || + (_tstate->jit_tracer_state.prev_state.instr->op.code == BINARY_OP_INPLACE_ADD_UNICODE && + opcode == STORE_FAST)) { + _tstate->jit_tracer_state.prev_state.instr_is_super = true; + } + else { + _tstate->jit_tracer_state.prev_state.instr = next_instr; + } + PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable); + if (_tstate->jit_tracer_state.prev_state.instr_code != (PyCodeObject *)prev_code) { + Py_SETREF(_tstate->jit_tracer_state.prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code))); + } + + _tstate->jit_tracer_state.prev_state.instr_frame = frame; + _tstate->jit_tracer_state.prev_state.instr_oparg = oparg; + _tstate->jit_tracer_state.prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + (&next_instr[1])->counter = trigger_backoff_counter(); + } + DISPATCH_GOTO_NON_TRACING(); +#else + Py_FatalError("JIT label executed in non-jit build."); +#endif + } + + label(stop_tracing) { +#if _Py_TIER2 + assert(IS_JIT_TRACING()); + int opcode = next_instr->op.code; + _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL, true); + LEAVE_TRACING(); + int err = stop_tracing_and_jit(tstate, frame); + ERROR_IF(err < 0); + DISPATCH_GOTO_NON_TRACING(); +#else + Py_FatalError("JIT label executed in non-jit build."); +#endif + } // END BYTECODES // diff --git a/Python/ceval.c b/Python/ceval.c index 07d21575e3a266..b76c9ec28119d5 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1004,6 +1004,8 @@ static const _Py_CODEUNIT _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS[] = { { .op.code = RESUME, .op.arg = RESUME_OPARG_DEPTH1_MASK | RESUME_AT_FUNC_START } }; +const _Py_CODEUNIT *_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR = (_Py_CODEUNIT*)&_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS; + #ifdef Py_DEBUG extern void _PyUOpPrint(const _PyUOpInstruction *uop); #endif @@ -1051,6 +1053,43 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) } } +#if _Py_TIER2 +// 0 for success, -1 for error. +static int +stop_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame) +{ + int _is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); + int err = 0; + if (!_PyErr_Occurred(tstate) && !_is_sys_tracing) { + err = _PyOptimizer_Optimize(frame, tstate); + } + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + // Deal with backoffs + _PyExitData *exit = _tstate->jit_tracer_state.initial_state.exit; + if (exit == NULL) { + // We hold a strong reference to the code object, so the instruction won't be freed. + if (err <= 0) { + _Py_BackoffCounter counter = _tstate->jit_tracer_state.initial_state.jump_backward_instr[1].counter; + _tstate->jit_tracer_state.initial_state.jump_backward_instr[1].counter = restart_backoff_counter(counter); + } + else { + _tstate->jit_tracer_state.initial_state.jump_backward_instr[1].counter = initial_jump_backoff_counter(); + } + } + else { + // Likewise, we hold a strong reference to the executor containing this exit, so the exit is guaranteed + // to be valid to access. + if (err <= 0) { + exit->temperature = restart_backoff_counter(exit->temperature); + } + else { + exit->temperature = initial_temperature_backoff_counter(); + } + } + _PyJit_FinalizeTracing(tstate); + return err; +} +#endif /* _PyEval_EvalFrameDefault is too large to optimize for speed with PGO on MSVC. */ @@ -1180,9 +1219,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int stack_pointer = _PyFrame_GetStackPointer(frame); #if _Py_TAIL_CALL_INTERP # if Py_STATS - return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, instruction_funcptr_table, 0, lastopcode); + return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, instruction_funcptr_handler_table, 0, lastopcode); # else - return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, instruction_funcptr_table, 0); + return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, instruction_funcptr_handler_table, 0); # endif #else goto error; @@ -1191,9 +1230,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #if _Py_TAIL_CALL_INTERP # if Py_STATS - return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, instruction_funcptr_table, 0, lastopcode); + return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, instruction_funcptr_handler_table, 0, lastopcode); # else - return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, instruction_funcptr_table, 0); + return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, instruction_funcptr_handler_table, 0); # endif #else goto start_frame; @@ -1235,7 +1274,9 @@ _PyTier2Interpreter( tier2_start: next_uop = current_executor->trace; - assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT); + assert(next_uop->opcode == _START_EXECUTOR || + next_uop->opcode == _COLD_EXIT || + next_uop->opcode == _COLD_DYNAMIC_EXIT); #undef LOAD_IP #define LOAD_IP(UNUSED) (void)0 @@ -1259,7 +1300,9 @@ _PyTier2Interpreter( uint64_t trace_uop_execution_counter = 0; #endif - assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT); + assert(next_uop->opcode == _START_EXECUTOR || + next_uop->opcode == _COLD_EXIT || + next_uop->opcode == _COLD_DYNAMIC_EXIT); tier2_dispatch: for (;;) { uopcode = next_uop->opcode; diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index afdcbc563b2c60..05a2760671e847 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -93,11 +93,19 @@ # define Py_PRESERVE_NONE_CC __attribute__((preserve_none)) Py_PRESERVE_NONE_CC typedef PyObject* (*py_tail_call_funcptr)(TAIL_CALL_PARAMS); +# define DISPATCH_TABLE_VAR instruction_funcptr_table +# define DISPATCH_TABLE instruction_funcptr_handler_table +# define TRACING_DISPATCH_TABLE instruction_funcptr_tracing_table # define TARGET(op) Py_PRESERVE_NONE_CC PyObject *_TAIL_CALL_##op(TAIL_CALL_PARAMS) + # define DISPATCH_GOTO() \ do { \ Py_MUSTTAIL return (((py_tail_call_funcptr *)instruction_funcptr_table)[opcode])(TAIL_CALL_ARGS); \ } while (0) +# define DISPATCH_GOTO_NON_TRACING() \ + do { \ + Py_MUSTTAIL return (((py_tail_call_funcptr *)DISPATCH_TABLE)[opcode])(TAIL_CALL_ARGS); \ + } while (0) # define JUMP_TO_LABEL(name) \ do { \ Py_MUSTTAIL return (_TAIL_CALL_##name)(TAIL_CALL_ARGS); \ @@ -115,19 +123,36 @@ # endif # define LABEL(name) TARGET(name) #elif USE_COMPUTED_GOTOS +# define DISPATCH_TABLE_VAR opcode_targets +# define DISPATCH_TABLE opcode_targets_table +# define TRACING_DISPATCH_TABLE opcode_tracing_targets_table # define TARGET(op) TARGET_##op: # define DISPATCH_GOTO() goto *opcode_targets[opcode] +# define DISPATCH_GOTO_NON_TRACING() goto *DISPATCH_TABLE[opcode]; # define JUMP_TO_LABEL(name) goto name; # define JUMP_TO_PREDICTED(name) goto PREDICTED_##name; # define LABEL(name) name: #else # define TARGET(op) case op: TARGET_##op: # define DISPATCH_GOTO() goto dispatch_opcode +# define DISPATCH_GOTO_NON_TRACING() goto dispatch_opcode # define JUMP_TO_LABEL(name) goto name; # define JUMP_TO_PREDICTED(name) goto PREDICTED_##name; # define LABEL(name) name: #endif +#if (_Py_TAIL_CALL_INTERP || USE_COMPUTED_GOTOS) && _Py_TIER2 +# define IS_JIT_TRACING() (DISPATCH_TABLE_VAR == TRACING_DISPATCH_TABLE) +# define ENTER_TRACING() \ + DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE; +# define LEAVE_TRACING() \ + DISPATCH_TABLE_VAR = DISPATCH_TABLE; +#else +# define IS_JIT_TRACING() (0) +# define ENTER_TRACING() +# define LEAVE_TRACING() +#endif + /* PRE_DISPATCH_GOTO() does lltrace if enabled. Normally a no-op */ #ifdef Py_DEBUG #define PRE_DISPATCH_GOTO() if (frame->lltrace >= 5) { \ @@ -164,11 +189,19 @@ do { \ DISPATCH_GOTO(); \ } +#define DISPATCH_NON_TRACING() \ + { \ + assert(frame->stackpointer == NULL); \ + NEXTOPARG(); \ + PRE_DISPATCH_GOTO(); \ + DISPATCH_GOTO_NON_TRACING(); \ + } + #define DISPATCH_SAME_OPARG() \ { \ opcode = next_instr->op.code; \ PRE_DISPATCH_GOTO(); \ - DISPATCH_GOTO(); \ + DISPATCH_GOTO_NON_TRACING(); \ } #define DISPATCH_INLINED(NEW_FRAME) \ @@ -280,6 +313,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { /* This takes a uint16_t instead of a _Py_BackoffCounter, * because it is used directly on the cache entry in generated code, * which is always an integral type. */ +// Force re-specialization when tracing a side exit to get good side exits. #define ADAPTIVE_COUNTER_TRIGGERS(COUNTER) \ backoff_counter_triggers(forge_backoff_counter((COUNTER))) @@ -366,12 +400,19 @@ do { \ next_instr = _Py_jit_entry((EXECUTOR), frame, stack_pointer, tstate); \ frame = tstate->current_frame; \ stack_pointer = _PyFrame_GetStackPointer(frame); \ + int keep_tracing_bit = (uintptr_t)next_instr & 1; \ + next_instr = (_Py_CODEUNIT *)(((uintptr_t)next_instr) & (~1)); \ if (next_instr == NULL) { \ /* gh-140104: The exception handler expects frame->instr_ptr to after this_instr, not this_instr! */ \ next_instr = frame->instr_ptr + 1; \ JUMP_TO_LABEL(error); \ } \ + if (keep_tracing_bit) { \ + assert(((_PyThreadStateImpl *)tstate)->jit_tracer_state.prev_state.code_curr_size == 2); \ + ENTER_TRACING(); \ + DISPATCH_NON_TRACING(); \ + } \ DISPATCH(); \ } while (0) @@ -382,13 +423,23 @@ do { \ goto tier2_start; \ } while (0) -#define GOTO_TIER_ONE(TARGET) \ - do \ - { \ - tstate->current_executor = NULL; \ - OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); \ - _PyFrame_SetStackPointer(frame, stack_pointer); \ - return TARGET; \ +#define GOTO_TIER_ONE_SETUP \ + tstate->current_executor = NULL; \ + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); \ + _PyFrame_SetStackPointer(frame, stack_pointer); + +#define GOTO_TIER_ONE(TARGET) \ + do \ + { \ + GOTO_TIER_ONE_SETUP \ + return (_Py_CODEUNIT *)(TARGET); \ + } while (0) + +#define GOTO_TIER_ONE_CONTINUE_TRACING(TARGET) \ + do \ + { \ + GOTO_TIER_ONE_SETUP \ + return (_Py_CODEUNIT *)(((uintptr_t)(TARGET))| 1); \ } while (0) #define CURRENT_OPARG() (next_uop[-1].oparg) diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 9ce0a9f8a4d87b..7ba2e9d0d92999 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4189,6 +4189,8 @@ break; } + /* _JUMP_BACKWARD_NO_INTERRUPT is not a viable micro-op for tier 2 because it is replaced */ + case _GET_LEN: { _PyStackRef obj; _PyStackRef len; @@ -7108,16 +7110,18 @@ PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); _PyExitData *exit = (_PyExitData *)exit_p; #if defined(Py_DEBUG) && !defined(_Py_JIT) - _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; + const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER) + ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame)) + + exit->target; OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - if (frame->lltrace >= 2) { + if (frame->lltrace >= 3) { _PyFrame_SetStackPointer(frame, stack_pointer); printf("SIDE EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); - printf(", exit %tu, temp %d, target %d -> %s]\n", + printf(", exit %tu, temp %d, target %d -> %s, is_control_flow %d]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyFrame_GetBytecode(frame)), - _PyOpcode_OpName[target->op.code]); + _PyOpcode_OpName[target->op.code], exit->is_control_flow); stack_pointer = _PyFrame_GetStackPointer(frame); } #endif @@ -7126,6 +7130,28 @@ break; } + case _DYNAMIC_EXIT: { + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); + #if defined(Py_DEBUG) && !defined(_Py_JIT) + _PyExitData *exit = (_PyExitData *)exit_p; + _Py_CODEUNIT *target = frame->instr_ptr; + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + if (frame->lltrace >= 3) { + _PyFrame_SetStackPointer(frame, stack_pointer); + printf("DYNAMIC EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(", exit %tu, temp %d, target %d -> %s]\n", + exit - current_executor->exits, exit->temperature.value_and_backoff, + (int)(target - _PyFrame_GetBytecode(frame)), + _PyOpcode_OpName[target->op.code]); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + #endif + + GOTO_TIER_ONE(frame->instr_ptr); + break; + } + case _CHECK_VALIDITY: { if (!current_executor->vm_data.valid) { UOP_STAT_INC(uopcode, miss); @@ -7419,7 +7445,8 @@ } case _DEOPT: { - GOTO_TIER_ONE(_PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + GOTO_TIER_ONE((frame->owner == FRAME_OWNED_BY_INTERPRETER) + ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); break; } @@ -7460,37 +7487,101 @@ case _COLD_EXIT: { _PyExitData *exit = tstate->jit_exit; assert(exit != NULL); + assert(frame->owner < FRAME_OWNED_BY_INTERPRETER); _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; _Py_BackoffCounter temperature = exit->temperature; - if (!backoff_counter_triggers(temperature)) { - exit->temperature = advance_backoff_counter(temperature); - GOTO_TIER_ONE(target); - } _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { PyCodeObject *code = _PyFrame_GetCode(frame); executor = code->co_executors->executors[target->op.arg]; Py_INCREF(executor); + assert(tstate->jit_exit == exit); + exit->executor = executor; + TIER2_TO_TIER2(exit->executor); } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + if (!backoff_counter_triggers(temperature)) { + exit->temperature = advance_backoff_counter(temperature); + GOTO_TIER_ONE(target); + } _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); - stack_pointer = _PyFrame_GetStackPointer(frame); assert(tstate->current_executor == (PyObject *)previous_executor); - int chain_depth = previous_executor->vm_data.chain_depth + 1; - _PyFrame_SetStackPointer(frame, stack_pointer); - int optimized = _PyOptimizer_Optimize(frame, target, &executor, chain_depth); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (optimized <= 0) { - exit->temperature = restart_backoff_counter(temperature); - GOTO_TIER_ONE(optimized < 0 ? NULL : target); + int chain_depth = previous_executor->vm_data.chain_depth + !exit->is_control_flow; + int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, target->op.arg); + exit->temperature = restart_backoff_counter(exit->temperature); + if (succ) { + GOTO_TIER_ONE_CONTINUE_TRACING(target); } - exit->temperature = initial_temperature_backoff_counter(); + GOTO_TIER_ONE(target); } - assert(tstate->jit_exit == exit); - exit->executor = executor; - TIER2_TO_TIER2(exit->executor); break; } + case _COLD_DYNAMIC_EXIT: { + _Py_CODEUNIT *target = frame->instr_ptr; + GOTO_TIER_ONE(target); + break; + } + + case _GUARD_IP__PUSH_FRAME: { + #define OFFSET_OF__PUSH_FRAME ((0)) + PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF__PUSH_FRAME; + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += OFFSET_OF__PUSH_FRAME; + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + #undef OFFSET_OF__PUSH_FRAME + break; + } + + case _GUARD_IP_YIELD_VALUE: { + #define OFFSET_OF_YIELD_VALUE ((1+INLINE_CACHE_ENTRIES_SEND)) + PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_YIELD_VALUE; + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += OFFSET_OF_YIELD_VALUE; + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + #undef OFFSET_OF_YIELD_VALUE + break; + } + + case _GUARD_IP_RETURN_VALUE: { + #define OFFSET_OF_RETURN_VALUE ((frame->return_offset)) + PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_VALUE; + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += OFFSET_OF_RETURN_VALUE; + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + #undef OFFSET_OF_RETURN_VALUE + break; + } + + case _GUARD_IP_RETURN_GENERATOR: { + #define OFFSET_OF_RETURN_GENERATOR ((frame->return_offset)) + PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_GENERATOR; + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += OFFSET_OF_RETURN_GENERATOR; + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + #undef OFFSET_OF_RETURN_GENERATOR + break; + } + + #undef TIER_TWO diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 79328a7b725613..a984da6dc912a2 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5476,6 +5476,10 @@ INSTRUCTION_STATS(ENTER_EXECUTOR); opcode = ENTER_EXECUTOR; #ifdef _Py_TIER2 + if (IS_JIT_TRACING()) { + next_instr = this_instr; + JUMP_TO_LABEL(stop_tracing); + } PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; assert(executor->vm_data.index == INSTR_OFFSET() - 1); @@ -7589,7 +7593,7 @@ /* Skip 1 cache entry */ // _SPECIALIZE_JUMP_BACKWARD { - #if ENABLE_SPECIALIZATION_FT + #if ENABLE_SPECIALIZATION if (this_instr->op.code == JUMP_BACKWARD) { uint8_t desired = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, desired); @@ -7645,30 +7649,20 @@ { #ifdef _Py_TIER2 _Py_BackoffCounter counter = this_instr[1].counter; - if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT) { - _Py_CODEUNIT *start = this_instr; + if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && + this_instr->op.code == JUMP_BACKWARD_JIT && + next_instr->op.code != ENTER_EXECUTOR) { + _Py_CODEUNIT *insert_exec_at = this_instr; while (oparg > 255) { oparg >>= 8; - start--; + insert_exec_at--; } - _PyExecutorObject *executor; - _PyFrame_SetStackPointer(frame, stack_pointer); - int optimized = _PyOptimizer_Optimize(frame, start, &executor, 0); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (optimized <= 0) { - this_instr[1].counter = restart_backoff_counter(counter); - if (optimized < 0) { - JUMP_TO_LABEL(error); - } + int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, oparg); + if (succ) { + ENTER_TRACING(); } else { - _PyFrame_SetStackPointer(frame, stack_pointer); - this_instr[1].counter = initial_jump_backoff_counter(); - stack_pointer = _PyFrame_GetStackPointer(frame); - assert(tstate->current_executor == NULL); - assert(executor != tstate->interp->cold_executor); - tstate->jit_exit = NULL; - TIER1_TO_TIER2(executor); + this_instr[1].counter = restart_backoff_counter(counter); } } else { @@ -12265,5 +12259,75 @@ JUMP_TO_LABEL(error); DISPATCH(); } + LABEL(record_previous_inst) + { + #if _Py_TIER2 + assert(IS_JIT_TRACING()); + int opcode = next_instr->op.code; + bool stop_tracing = (opcode == WITH_EXCEPT_START || + opcode == RERAISE || opcode == CLEANUP_THROW || + opcode == PUSH_EXC_INFO || opcode == INTERPRETER_EXIT); + _PyFrame_SetStackPointer(frame, stack_pointer); + int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (full) { + LEAVE_TRACING(); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = stop_tracing_and_jit(tstate, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + DISPATCH_GOTO_NON_TRACING(); + } + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + if ((_tstate->jit_tracer_state.prev_state.instr->op.code == CALL_LIST_APPEND && + opcode == POP_TOP) || + (_tstate->jit_tracer_state.prev_state.instr->op.code == BINARY_OP_INPLACE_ADD_UNICODE && + opcode == STORE_FAST)) { + _tstate->jit_tracer_state.prev_state.instr_is_super = true; + } + else { + _tstate->jit_tracer_state.prev_state.instr = next_instr; + } + PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable); + if (_tstate->jit_tracer_state.prev_state.instr_code != (PyCodeObject *)prev_code) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_SETREF(_tstate->jit_tracer_state.prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code))); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + _tstate->jit_tracer_state.prev_state.instr_frame = frame; + _tstate->jit_tracer_state.prev_state.instr_oparg = oparg; + _tstate->jit_tracer_state.prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + (&next_instr[1])->counter = trigger_backoff_counter(); + } + DISPATCH_GOTO_NON_TRACING(); + #else + Py_FatalError("JIT label executed in non-jit build."); + #endif + } + + LABEL(stop_tracing) + { + #if _Py_TIER2 + assert(IS_JIT_TRACING()); + int opcode = next_instr->op.code; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL, true); + stack_pointer = _PyFrame_GetStackPointer(frame); + LEAVE_TRACING(); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = stop_tracing_and_jit(tstate, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + DISPATCH_GOTO_NON_TRACING(); + #else + Py_FatalError("JIT label executed in non-jit build."); + #endif + } + /* END LABELS */ #undef TIER_ONE diff --git a/Python/instrumentation.c b/Python/instrumentation.c index b4b2bc5dc69f9d..81e46a331e0b9e 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -18,6 +18,7 @@ #include "pycore_tuple.h" // _PyTuple_FromArraySteal() #include "opcode_ids.h" +#include "pycore_optimizer.h" /* Uncomment this to dump debugging output when assertions fail */ @@ -1785,6 +1786,7 @@ force_instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) _PyCode_Clear_Executors(code); } _Py_Executors_InvalidateDependency(interp, code, 1); + _PyJit_Tracer_InvalidateDependency(PyThreadState_GET(), code); #endif int code_len = (int)Py_SIZE(code); /* Exit early to avoid creating instrumentation diff --git a/Python/jit.c b/Python/jit.c index 279e1ce6a0d2e5..7ab0f8ddd430dd 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -604,7 +604,7 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], siz unsigned char *code = memory; state.trampolines.mem = memory + code_size; unsigned char *data = memory + code_size + state.trampolines.size + code_padding; - assert(trace[0].opcode == _START_EXECUTOR || trace[0].opcode == _COLD_EXIT); + assert(trace[0].opcode == _START_EXECUTOR || trace[0].opcode == _COLD_EXIT || trace[0].opcode == _COLD_DYNAMIC_EXIT); for (size_t i = 0; i < length; i++) { const _PyUOpInstruction *instruction = &trace[i]; group = &stencil_groups[instruction->opcode]; diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 6dd443e1655ed0..1b9196503b570b 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -257,8 +257,270 @@ static void *opcode_targets_table[256] = { &&TARGET_INSTRUMENTED_LINE, &&TARGET_ENTER_EXECUTOR, }; +#if _Py_TIER2 +static void *opcode_tracing_targets_table[256] = { + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, +}; +#endif #else /* _Py_TAIL_CALL_INTERP */ -static py_tail_call_funcptr instruction_funcptr_table[256]; +static py_tail_call_funcptr instruction_funcptr_handler_table[256]; + +static py_tail_call_funcptr instruction_funcptr_tracing_table[256]; Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS); @@ -266,6 +528,8 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_record_previous_inst(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_stop_tracing(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS); @@ -503,7 +767,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) JUMP_TO_LABEL(error); } -static py_tail_call_funcptr instruction_funcptr_table[256] = { +static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [BINARY_OP] = _TAIL_CALL_BINARY_OP, [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_BINARY_OP_ADD_FLOAT, [BINARY_OP_ADD_INT] = _TAIL_CALL_BINARY_OP_ADD_INT, @@ -761,4 +1025,262 @@ static py_tail_call_funcptr instruction_funcptr_table[256] = { [232] = _TAIL_CALL_UNKNOWN_OPCODE, [233] = _TAIL_CALL_UNKNOWN_OPCODE, }; +static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { + [BINARY_OP] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_ADD_INT] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_ADD_UNICODE] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_EXTEND] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_INPLACE_ADD_UNICODE] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_MULTIPLY_FLOAT] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_MULTIPLY_INT] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_SUBSCR_DICT] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_SUBSCR_GETITEM] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_SUBSCR_LIST_INT] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_SUBSCR_LIST_SLICE] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_SUBSCR_STR_INT] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_SUBSCR_TUPLE_INT] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_SUBTRACT_FLOAT] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_SUBTRACT_INT] = _TAIL_CALL_record_previous_inst, + [BINARY_SLICE] = _TAIL_CALL_record_previous_inst, + [BUILD_INTERPOLATION] = _TAIL_CALL_record_previous_inst, + [BUILD_LIST] = _TAIL_CALL_record_previous_inst, + [BUILD_MAP] = _TAIL_CALL_record_previous_inst, + [BUILD_SET] = _TAIL_CALL_record_previous_inst, + [BUILD_SLICE] = _TAIL_CALL_record_previous_inst, + [BUILD_STRING] = _TAIL_CALL_record_previous_inst, + [BUILD_TEMPLATE] = _TAIL_CALL_record_previous_inst, + [BUILD_TUPLE] = _TAIL_CALL_record_previous_inst, + [CACHE] = _TAIL_CALL_record_previous_inst, + [CALL] = _TAIL_CALL_record_previous_inst, + [CALL_ALLOC_AND_ENTER_INIT] = _TAIL_CALL_record_previous_inst, + [CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_record_previous_inst, + [CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_record_previous_inst, + [CALL_BUILTIN_CLASS] = _TAIL_CALL_record_previous_inst, + [CALL_BUILTIN_FAST] = _TAIL_CALL_record_previous_inst, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = _TAIL_CALL_record_previous_inst, + [CALL_BUILTIN_O] = _TAIL_CALL_record_previous_inst, + [CALL_FUNCTION_EX] = _TAIL_CALL_record_previous_inst, + [CALL_INTRINSIC_1] = _TAIL_CALL_record_previous_inst, + [CALL_INTRINSIC_2] = _TAIL_CALL_record_previous_inst, + [CALL_ISINSTANCE] = _TAIL_CALL_record_previous_inst, + [CALL_KW] = _TAIL_CALL_record_previous_inst, + [CALL_KW_BOUND_METHOD] = _TAIL_CALL_record_previous_inst, + [CALL_KW_NON_PY] = _TAIL_CALL_record_previous_inst, + [CALL_KW_PY] = _TAIL_CALL_record_previous_inst, + [CALL_LEN] = _TAIL_CALL_record_previous_inst, + [CALL_LIST_APPEND] = _TAIL_CALL_record_previous_inst, + [CALL_METHOD_DESCRIPTOR_FAST] = _TAIL_CALL_record_previous_inst, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = _TAIL_CALL_record_previous_inst, + [CALL_METHOD_DESCRIPTOR_NOARGS] = _TAIL_CALL_record_previous_inst, + [CALL_METHOD_DESCRIPTOR_O] = _TAIL_CALL_record_previous_inst, + [CALL_NON_PY_GENERAL] = _TAIL_CALL_record_previous_inst, + [CALL_PY_EXACT_ARGS] = _TAIL_CALL_record_previous_inst, + [CALL_PY_GENERAL] = _TAIL_CALL_record_previous_inst, + [CALL_STR_1] = _TAIL_CALL_record_previous_inst, + [CALL_TUPLE_1] = _TAIL_CALL_record_previous_inst, + [CALL_TYPE_1] = _TAIL_CALL_record_previous_inst, + [CHECK_EG_MATCH] = _TAIL_CALL_record_previous_inst, + [CHECK_EXC_MATCH] = _TAIL_CALL_record_previous_inst, + [CLEANUP_THROW] = _TAIL_CALL_record_previous_inst, + [COMPARE_OP] = _TAIL_CALL_record_previous_inst, + [COMPARE_OP_FLOAT] = _TAIL_CALL_record_previous_inst, + [COMPARE_OP_INT] = _TAIL_CALL_record_previous_inst, + [COMPARE_OP_STR] = _TAIL_CALL_record_previous_inst, + [CONTAINS_OP] = _TAIL_CALL_record_previous_inst, + [CONTAINS_OP_DICT] = _TAIL_CALL_record_previous_inst, + [CONTAINS_OP_SET] = _TAIL_CALL_record_previous_inst, + [CONVERT_VALUE] = _TAIL_CALL_record_previous_inst, + [COPY] = _TAIL_CALL_record_previous_inst, + [COPY_FREE_VARS] = _TAIL_CALL_record_previous_inst, + [DELETE_ATTR] = _TAIL_CALL_record_previous_inst, + [DELETE_DEREF] = _TAIL_CALL_record_previous_inst, + [DELETE_FAST] = _TAIL_CALL_record_previous_inst, + [DELETE_GLOBAL] = _TAIL_CALL_record_previous_inst, + [DELETE_NAME] = _TAIL_CALL_record_previous_inst, + [DELETE_SUBSCR] = _TAIL_CALL_record_previous_inst, + [DICT_MERGE] = _TAIL_CALL_record_previous_inst, + [DICT_UPDATE] = _TAIL_CALL_record_previous_inst, + [END_ASYNC_FOR] = _TAIL_CALL_record_previous_inst, + [END_FOR] = _TAIL_CALL_record_previous_inst, + [END_SEND] = _TAIL_CALL_record_previous_inst, + [ENTER_EXECUTOR] = _TAIL_CALL_record_previous_inst, + [EXIT_INIT_CHECK] = _TAIL_CALL_record_previous_inst, + [EXTENDED_ARG] = _TAIL_CALL_record_previous_inst, + [FORMAT_SIMPLE] = _TAIL_CALL_record_previous_inst, + [FORMAT_WITH_SPEC] = _TAIL_CALL_record_previous_inst, + [FOR_ITER] = _TAIL_CALL_record_previous_inst, + [FOR_ITER_GEN] = _TAIL_CALL_record_previous_inst, + [FOR_ITER_LIST] = _TAIL_CALL_record_previous_inst, + [FOR_ITER_RANGE] = _TAIL_CALL_record_previous_inst, + [FOR_ITER_TUPLE] = _TAIL_CALL_record_previous_inst, + [GET_AITER] = _TAIL_CALL_record_previous_inst, + [GET_ANEXT] = _TAIL_CALL_record_previous_inst, + [GET_AWAITABLE] = _TAIL_CALL_record_previous_inst, + [GET_ITER] = _TAIL_CALL_record_previous_inst, + [GET_LEN] = _TAIL_CALL_record_previous_inst, + [GET_YIELD_FROM_ITER] = _TAIL_CALL_record_previous_inst, + [IMPORT_FROM] = _TAIL_CALL_record_previous_inst, + [IMPORT_NAME] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_CALL] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_CALL_FUNCTION_EX] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_CALL_KW] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_END_ASYNC_FOR] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_END_FOR] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_END_SEND] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_FOR_ITER] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_INSTRUCTION] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_JUMP_BACKWARD] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_JUMP_FORWARD] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_LINE] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_LOAD_SUPER_ATTR] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_NOT_TAKEN] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_POP_ITER] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_POP_JUMP_IF_NONE] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_RESUME] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_RETURN_VALUE] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_YIELD_VALUE] = _TAIL_CALL_record_previous_inst, + [INTERPRETER_EXIT] = _TAIL_CALL_record_previous_inst, + [IS_OP] = _TAIL_CALL_record_previous_inst, + [JUMP_BACKWARD] = _TAIL_CALL_record_previous_inst, + [JUMP_BACKWARD_JIT] = _TAIL_CALL_record_previous_inst, + [JUMP_BACKWARD_NO_INTERRUPT] = _TAIL_CALL_record_previous_inst, + [JUMP_BACKWARD_NO_JIT] = _TAIL_CALL_record_previous_inst, + [JUMP_FORWARD] = _TAIL_CALL_record_previous_inst, + [LIST_APPEND] = _TAIL_CALL_record_previous_inst, + [LIST_EXTEND] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_CLASS] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_INSTANCE_VALUE] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_METHOD_LAZY_DICT] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_METHOD_NO_DICT] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_METHOD_WITH_VALUES] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_MODULE] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_PROPERTY] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_SLOT] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_WITH_HINT] = _TAIL_CALL_record_previous_inst, + [LOAD_BUILD_CLASS] = _TAIL_CALL_record_previous_inst, + [LOAD_COMMON_CONSTANT] = _TAIL_CALL_record_previous_inst, + [LOAD_CONST] = _TAIL_CALL_record_previous_inst, + [LOAD_DEREF] = _TAIL_CALL_record_previous_inst, + [LOAD_FAST] = _TAIL_CALL_record_previous_inst, + [LOAD_FAST_AND_CLEAR] = _TAIL_CALL_record_previous_inst, + [LOAD_FAST_BORROW] = _TAIL_CALL_record_previous_inst, + [LOAD_FAST_BORROW_LOAD_FAST_BORROW] = _TAIL_CALL_record_previous_inst, + [LOAD_FAST_CHECK] = _TAIL_CALL_record_previous_inst, + [LOAD_FAST_LOAD_FAST] = _TAIL_CALL_record_previous_inst, + [LOAD_FROM_DICT_OR_DEREF] = _TAIL_CALL_record_previous_inst, + [LOAD_FROM_DICT_OR_GLOBALS] = _TAIL_CALL_record_previous_inst, + [LOAD_GLOBAL] = _TAIL_CALL_record_previous_inst, + [LOAD_GLOBAL_BUILTIN] = _TAIL_CALL_record_previous_inst, + [LOAD_GLOBAL_MODULE] = _TAIL_CALL_record_previous_inst, + [LOAD_LOCALS] = _TAIL_CALL_record_previous_inst, + [LOAD_NAME] = _TAIL_CALL_record_previous_inst, + [LOAD_SMALL_INT] = _TAIL_CALL_record_previous_inst, + [LOAD_SPECIAL] = _TAIL_CALL_record_previous_inst, + [LOAD_SUPER_ATTR] = _TAIL_CALL_record_previous_inst, + [LOAD_SUPER_ATTR_ATTR] = _TAIL_CALL_record_previous_inst, + [LOAD_SUPER_ATTR_METHOD] = _TAIL_CALL_record_previous_inst, + [MAKE_CELL] = _TAIL_CALL_record_previous_inst, + [MAKE_FUNCTION] = _TAIL_CALL_record_previous_inst, + [MAP_ADD] = _TAIL_CALL_record_previous_inst, + [MATCH_CLASS] = _TAIL_CALL_record_previous_inst, + [MATCH_KEYS] = _TAIL_CALL_record_previous_inst, + [MATCH_MAPPING] = _TAIL_CALL_record_previous_inst, + [MATCH_SEQUENCE] = _TAIL_CALL_record_previous_inst, + [NOP] = _TAIL_CALL_record_previous_inst, + [NOT_TAKEN] = _TAIL_CALL_record_previous_inst, + [POP_EXCEPT] = _TAIL_CALL_record_previous_inst, + [POP_ITER] = _TAIL_CALL_record_previous_inst, + [POP_JUMP_IF_FALSE] = _TAIL_CALL_record_previous_inst, + [POP_JUMP_IF_NONE] = _TAIL_CALL_record_previous_inst, + [POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_record_previous_inst, + [POP_JUMP_IF_TRUE] = _TAIL_CALL_record_previous_inst, + [POP_TOP] = _TAIL_CALL_record_previous_inst, + [PUSH_EXC_INFO] = _TAIL_CALL_record_previous_inst, + [PUSH_NULL] = _TAIL_CALL_record_previous_inst, + [RAISE_VARARGS] = _TAIL_CALL_record_previous_inst, + [RERAISE] = _TAIL_CALL_record_previous_inst, + [RESERVED] = _TAIL_CALL_record_previous_inst, + [RESUME] = _TAIL_CALL_record_previous_inst, + [RESUME_CHECK] = _TAIL_CALL_record_previous_inst, + [RETURN_GENERATOR] = _TAIL_CALL_record_previous_inst, + [RETURN_VALUE] = _TAIL_CALL_record_previous_inst, + [SEND] = _TAIL_CALL_record_previous_inst, + [SEND_GEN] = _TAIL_CALL_record_previous_inst, + [SETUP_ANNOTATIONS] = _TAIL_CALL_record_previous_inst, + [SET_ADD] = _TAIL_CALL_record_previous_inst, + [SET_FUNCTION_ATTRIBUTE] = _TAIL_CALL_record_previous_inst, + [SET_UPDATE] = _TAIL_CALL_record_previous_inst, + [STORE_ATTR] = _TAIL_CALL_record_previous_inst, + [STORE_ATTR_INSTANCE_VALUE] = _TAIL_CALL_record_previous_inst, + [STORE_ATTR_SLOT] = _TAIL_CALL_record_previous_inst, + [STORE_ATTR_WITH_HINT] = _TAIL_CALL_record_previous_inst, + [STORE_DEREF] = _TAIL_CALL_record_previous_inst, + [STORE_FAST] = _TAIL_CALL_record_previous_inst, + [STORE_FAST_LOAD_FAST] = _TAIL_CALL_record_previous_inst, + [STORE_FAST_STORE_FAST] = _TAIL_CALL_record_previous_inst, + [STORE_GLOBAL] = _TAIL_CALL_record_previous_inst, + [STORE_NAME] = _TAIL_CALL_record_previous_inst, + [STORE_SLICE] = _TAIL_CALL_record_previous_inst, + [STORE_SUBSCR] = _TAIL_CALL_record_previous_inst, + [STORE_SUBSCR_DICT] = _TAIL_CALL_record_previous_inst, + [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_record_previous_inst, + [SWAP] = _TAIL_CALL_record_previous_inst, + [TO_BOOL] = _TAIL_CALL_record_previous_inst, + [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_record_previous_inst, + [TO_BOOL_BOOL] = _TAIL_CALL_record_previous_inst, + [TO_BOOL_INT] = _TAIL_CALL_record_previous_inst, + [TO_BOOL_LIST] = _TAIL_CALL_record_previous_inst, + [TO_BOOL_NONE] = _TAIL_CALL_record_previous_inst, + [TO_BOOL_STR] = _TAIL_CALL_record_previous_inst, + [UNARY_INVERT] = _TAIL_CALL_record_previous_inst, + [UNARY_NEGATIVE] = _TAIL_CALL_record_previous_inst, + [UNARY_NOT] = _TAIL_CALL_record_previous_inst, + [UNPACK_EX] = _TAIL_CALL_record_previous_inst, + [UNPACK_SEQUENCE] = _TAIL_CALL_record_previous_inst, + [UNPACK_SEQUENCE_LIST] = _TAIL_CALL_record_previous_inst, + [UNPACK_SEQUENCE_TUPLE] = _TAIL_CALL_record_previous_inst, + [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_record_previous_inst, + [WITH_EXCEPT_START] = _TAIL_CALL_record_previous_inst, + [YIELD_VALUE] = _TAIL_CALL_record_previous_inst, + [121] = _TAIL_CALL_UNKNOWN_OPCODE, + [122] = _TAIL_CALL_UNKNOWN_OPCODE, + [123] = _TAIL_CALL_UNKNOWN_OPCODE, + [124] = _TAIL_CALL_UNKNOWN_OPCODE, + [125] = _TAIL_CALL_UNKNOWN_OPCODE, + [126] = _TAIL_CALL_UNKNOWN_OPCODE, + [127] = _TAIL_CALL_UNKNOWN_OPCODE, + [210] = _TAIL_CALL_UNKNOWN_OPCODE, + [211] = _TAIL_CALL_UNKNOWN_OPCODE, + [212] = _TAIL_CALL_UNKNOWN_OPCODE, + [213] = _TAIL_CALL_UNKNOWN_OPCODE, + [214] = _TAIL_CALL_UNKNOWN_OPCODE, + [215] = _TAIL_CALL_UNKNOWN_OPCODE, + [216] = _TAIL_CALL_UNKNOWN_OPCODE, + [217] = _TAIL_CALL_UNKNOWN_OPCODE, + [218] = _TAIL_CALL_UNKNOWN_OPCODE, + [219] = _TAIL_CALL_UNKNOWN_OPCODE, + [220] = _TAIL_CALL_UNKNOWN_OPCODE, + [221] = _TAIL_CALL_UNKNOWN_OPCODE, + [222] = _TAIL_CALL_UNKNOWN_OPCODE, + [223] = _TAIL_CALL_UNKNOWN_OPCODE, + [224] = _TAIL_CALL_UNKNOWN_OPCODE, + [225] = _TAIL_CALL_UNKNOWN_OPCODE, + [226] = _TAIL_CALL_UNKNOWN_OPCODE, + [227] = _TAIL_CALL_UNKNOWN_OPCODE, + [228] = _TAIL_CALL_UNKNOWN_OPCODE, + [229] = _TAIL_CALL_UNKNOWN_OPCODE, + [230] = _TAIL_CALL_UNKNOWN_OPCODE, + [231] = _TAIL_CALL_UNKNOWN_OPCODE, + [232] = _TAIL_CALL_UNKNOWN_OPCODE, + [233] = _TAIL_CALL_UNKNOWN_OPCODE, +}; #endif /* _Py_TAIL_CALL_INTERP */ diff --git a/Python/optimizer.c b/Python/optimizer.c index 3b7e2dafab85bb..65007a256d0c3b 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -29,11 +29,24 @@ #define MAX_EXECUTORS_SIZE 256 +// Trace too short, no progress: +// _START_EXECUTOR +// _MAKE_WARM +// _CHECK_VALIDITY +// _SET_IP +// is 4-5 instructions. +#define CODE_SIZE_NO_PROGRESS 5 +// We start with _START_EXECUTOR, _MAKE_WARM +#define CODE_SIZE_EMPTY 2 + #define _PyExecutorObject_CAST(op) ((_PyExecutorObject *)(op)) static bool has_space_for_executor(PyCodeObject *code, _Py_CODEUNIT *instr) { + if (code == (PyCodeObject *)&_Py_InitCleanup) { + return false; + } if (instr->op.code == ENTER_EXECUTOR) { return true; } @@ -100,11 +113,11 @@ insert_executor(PyCodeObject *code, _Py_CODEUNIT *instr, int index, _PyExecutorO } static _PyExecutorObject * -make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies); +make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies, int chain_depth); static int -uop_optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *instr, - _PyExecutorObject **exec_ptr, int curr_stackentries, +uop_optimize(_PyInterpreterFrame *frame, PyThreadState *tstate, + _PyExecutorObject **exec_ptr, bool progress_needed); /* Returns 1 if optimized, 0 if not optimized, and -1 for an error. @@ -113,10 +126,10 @@ uop_optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *instr, // gh-137573: inlining this function causes stack overflows Py_NO_INLINE int _PyOptimizer_Optimize( - _PyInterpreterFrame *frame, _Py_CODEUNIT *start, - _PyExecutorObject **executor_ptr, int chain_depth) + _PyInterpreterFrame *frame, PyThreadState *tstate) { - _PyStackRef *stack_pointer = frame->stackpointer; + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + int chain_depth = _tstate->jit_tracer_state.initial_state.chain_depth; PyInterpreterState *interp = _PyInterpreterState_GET(); if (!interp->jit) { // gh-140936: It is possible that interp->jit will become false during @@ -126,7 +139,9 @@ _PyOptimizer_Optimize( return 0; } assert(!interp->compiling); + assert(_tstate->jit_tracer_state.initial_state.stack_depth >= 0); #ifndef Py_GIL_DISABLED + assert(_tstate->jit_tracer_state.initial_state.func != NULL); interp->compiling = true; // The first executor in a chain and the MAX_CHAIN_DEPTH'th executor *must* // make progress in order to avoid infinite loops or excessively-long @@ -134,18 +149,24 @@ _PyOptimizer_Optimize( // this is true, since a deopt won't infinitely re-enter the executor: chain_depth %= MAX_CHAIN_DEPTH; bool progress_needed = chain_depth == 0; - PyCodeObject *code = _PyFrame_GetCode(frame); - assert(PyCode_Check(code)); + PyCodeObject *code = (PyCodeObject *)_tstate->jit_tracer_state.initial_state.code; + _Py_CODEUNIT *start = _tstate->jit_tracer_state.initial_state.start_instr; if (progress_needed && !has_space_for_executor(code, start)) { interp->compiling = false; return 0; } - int err = uop_optimize(frame, start, executor_ptr, (int)(stack_pointer - _PyFrame_Stackbase(frame)), progress_needed); + // One of our dependencies while tracing was invalidated. Not worth compiling. + if (!_tstate->jit_tracer_state.prev_state.dependencies_still_valid) { + interp->compiling = false; + return 0; + } + _PyExecutorObject *executor; + int err = uop_optimize(frame, tstate, &executor, progress_needed); if (err <= 0) { interp->compiling = false; return err; } - assert(*executor_ptr != NULL); + assert(executor != NULL); if (progress_needed) { int index = get_index_for_executor(code, start); if (index < 0) { @@ -155,17 +176,21 @@ _PyOptimizer_Optimize( * If an optimizer has already produced an executor, * it might get confused by the executor disappearing, * but there is not much we can do about that here. */ - Py_DECREF(*executor_ptr); + Py_DECREF(executor); interp->compiling = false; return 0; } - insert_executor(code, start, index, *executor_ptr); + insert_executor(code, start, index, executor); } else { - (*executor_ptr)->vm_data.code = NULL; + executor->vm_data.code = NULL; + } + _PyExitData *exit = _tstate->jit_tracer_state.initial_state.exit; + if (exit != NULL) { + exit->executor = executor; } - (*executor_ptr)->vm_data.chain_depth = chain_depth; - assert((*executor_ptr)->vm_data.valid); + executor->vm_data.chain_depth = chain_depth; + assert(executor->vm_data.valid); interp->compiling = false; return 1; #else @@ -474,6 +499,14 @@ BRANCH_TO_GUARD[4][2] = { [POP_JUMP_IF_NOT_NONE - POP_JUMP_IF_FALSE][1] = _GUARD_IS_NOT_NONE_POP, }; +static const uint16_t +guard_ip_uop[MAX_UOP_ID + 1] = { + [_PUSH_FRAME] = _GUARD_IP__PUSH_FRAME, + [_RETURN_GENERATOR] = _GUARD_IP_RETURN_GENERATOR, + [_RETURN_VALUE] = _GUARD_IP_RETURN_VALUE, + [_YIELD_VALUE] = _GUARD_IP_YIELD_VALUE, +}; + #define CONFIDENCE_RANGE 1000 #define CONFIDENCE_CUTOFF 333 @@ -530,64 +563,19 @@ add_to_trace( DPRINTF(2, "No room for %s (need %d, got %d)\n", \ (opname), (n), max_length - trace_length); \ OPT_STAT_INC(trace_too_long); \ - goto done; \ - } - -// Reserve space for N uops, plus 3 for _SET_IP, _CHECK_VALIDITY and _EXIT_TRACE -#define RESERVE(needed) RESERVE_RAW((needed) + 3, _PyUOpName(opcode)) - -// Trace stack operations (used by _PUSH_FRAME, _RETURN_VALUE) -#define TRACE_STACK_PUSH() \ - if (trace_stack_depth >= TRACE_STACK_SIZE) { \ - DPRINTF(2, "Trace stack overflow\n"); \ - OPT_STAT_INC(trace_stack_overflow); \ - return 0; \ - } \ - assert(func == NULL || func->func_code == (PyObject *)code); \ - trace_stack[trace_stack_depth].func = func; \ - trace_stack[trace_stack_depth].code = code; \ - trace_stack[trace_stack_depth].instr = instr; \ - trace_stack_depth++; -#define TRACE_STACK_POP() \ - if (trace_stack_depth <= 0) { \ - Py_FatalError("Trace stack underflow\n"); \ - } \ - trace_stack_depth--; \ - func = trace_stack[trace_stack_depth].func; \ - code = trace_stack[trace_stack_depth].code; \ - assert(func == NULL || func->func_code == (PyObject *)code); \ - instr = trace_stack[trace_stack_depth].instr; - -/* Returns the length of the trace on success, - * 0 if it failed to produce a worthwhile trace, - * and -1 on an error. + goto full; \ + } + + +/* Returns 1 on success (added to trace), 0 on trace end. */ -static int -translate_bytecode_to_trace( +int +_PyJit_translate_single_bytecode_to_trace( + PyThreadState *tstate, _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr, - _PyUOpInstruction *trace, - int buffer_size, - _PyBloomFilter *dependencies, bool progress_needed) + _Py_CODEUNIT *next_instr, + bool stop_tracing) { - bool first = true; - PyCodeObject *code = _PyFrame_GetCode(frame); - PyFunctionObject *func = _PyFrame_GetFunction(frame); - assert(PyFunction_Check(func)); - PyCodeObject *initial_code = code; - _Py_BloomFilter_Add(dependencies, initial_code); - _Py_CODEUNIT *initial_instr = instr; - int trace_length = 0; - // Leave space for possible trailing _EXIT_TRACE - int max_length = buffer_size-2; - struct { - PyFunctionObject *func; - PyCodeObject *code; - _Py_CODEUNIT *instr; - } trace_stack[TRACE_STACK_SIZE]; - int trace_stack_depth = 0; - int confidence = CONFIDENCE_RANGE; // Adjusted by branch instructions - bool jump_seen = false; #ifdef Py_DEBUG char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); @@ -596,410 +584,468 @@ translate_bytecode_to_trace( lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that } #endif - - DPRINTF(2, - "Optimizing %s (%s:%d) at byte offset %d\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - 2 * INSTR_IP(initial_instr, code)); - ADD_TO_TRACE(_START_EXECUTOR, 0, (uintptr_t)instr, INSTR_IP(instr, code)); - ADD_TO_TRACE(_MAKE_WARM, 0, 0, 0); + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + PyCodeObject *old_code = _tstate->jit_tracer_state.prev_state.instr_code; + bool progress_needed = (_tstate->jit_tracer_state.initial_state.chain_depth % MAX_CHAIN_DEPTH) == 0; + _PyBloomFilter *dependencies = &_tstate->jit_tracer_state.prev_state.dependencies; + int trace_length = _tstate->jit_tracer_state.prev_state.code_curr_size; + _PyUOpInstruction *trace = _tstate->jit_tracer_state.code_buffer; + int max_length = _tstate->jit_tracer_state.prev_state.code_max_size; + + _Py_CODEUNIT *this_instr = _tstate->jit_tracer_state.prev_state.instr; + _Py_CODEUNIT *target_instr = this_instr; uint32_t target = 0; - for (;;) { - target = INSTR_IP(instr, code); - // One for possible _DEOPT, one because _CHECK_VALIDITY itself might _DEOPT - max_length-=2; - uint32_t opcode = instr->op.code; - uint32_t oparg = instr->op.arg; - - if (!first && instr == initial_instr) { - // We have looped around to the start: - RESERVE(1); - ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0); - goto done; + target = Py_IsNone((PyObject *)old_code) + ? (int)(target_instr - _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR) + : INSTR_IP(target_instr, old_code); + + // Rewind EXTENDED_ARG so that we see the whole thing. + // We must point to the first EXTENDED_ARG when deopting. + int oparg = _tstate->jit_tracer_state.prev_state.instr_oparg; + int opcode = this_instr->op.code; + int rewind_oparg = oparg; + while (rewind_oparg > 255) { + rewind_oparg >>= 8; + target--; + } + + int old_stack_level = _tstate->jit_tracer_state.prev_state.instr_stacklevel; + + // Strange control-flow + bool has_dynamic_jump_taken = OPCODE_HAS_UNPREDICTABLE_JUMP(opcode) && + (next_instr != this_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]); + + /* Special case the first instruction, + * so that we can guarantee forward progress */ + if (progress_needed && _tstate->jit_tracer_state.prev_state.code_curr_size < CODE_SIZE_NO_PROGRESS) { + if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) { + opcode = _PyOpcode_Deopt[opcode]; } + assert(!OPCODE_HAS_EXIT(opcode)); + assert(!OPCODE_HAS_DEOPT(opcode)); + } - DPRINTF(2, "%d: %s(%d)\n", target, _PyOpcode_OpName[opcode], oparg); + bool needs_guard_ip = OPCODE_HAS_NEEDS_GUARD_IP(opcode); + if (has_dynamic_jump_taken && !needs_guard_ip) { + DPRINTF(2, "Unsupported: dynamic jump taken %s\n", _PyOpcode_OpName[opcode]); + goto unsupported; + } - if (opcode == EXTENDED_ARG) { - instr++; - opcode = instr->op.code; - oparg = (oparg << 8) | instr->op.arg; - if (opcode == EXTENDED_ARG) { - instr--; - goto done; + int is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); + if (is_sys_tracing) { + goto full; + } + + if (stop_tracing) { + ADD_TO_TRACE(_DEOPT, 0, 0, target); + goto done; + } + + DPRINTF(2, "%p %d: %s(%d) %d %d\n", old_code, target, _PyOpcode_OpName[opcode], oparg, needs_guard_ip, old_stack_level); + +#ifdef Py_DEBUG + if (oparg > 255) { + assert(_Py_GetBaseCodeUnit(old_code, target).op.code == EXTENDED_ARG); + } +#endif + + // Skip over super instructions. + if (_tstate->jit_tracer_state.prev_state.instr_is_super) { + _tstate->jit_tracer_state.prev_state.instr_is_super = false; + return 1; + } + + if (opcode == ENTER_EXECUTOR) { + goto full; + } + + if (!_tstate->jit_tracer_state.prev_state.dependencies_still_valid) { + goto done; + } + + // This happens when a recursive call happens that we can't trace. Such as Python -> C -> Python calls + // If we haven't guarded the IP, then it's untraceable. + if (frame != _tstate->jit_tracer_state.prev_state.instr_frame && !needs_guard_ip) { + DPRINTF(2, "Unsupported: unguardable jump taken\n"); + goto unsupported; + } + + if (oparg > 0xFFFF) { + DPRINTF(2, "Unsupported: oparg too large\n"); + goto unsupported; + } + + // TODO (gh-140277): The constituent use one extra stack slot. So we need to check for headroom. + if (opcode == BINARY_OP_SUBSCR_GETITEM && old_stack_level + 1 > old_code->co_stacksize) { + unsupported: + { + // Rewind to previous instruction and replace with _EXIT_TRACE. + _PyUOpInstruction *curr = &trace[trace_length-1]; + while (curr->opcode != _SET_IP && trace_length > 2) { + trace_length--; + curr = &trace[trace_length-1]; + } + assert(curr->opcode == _SET_IP || trace_length == 2); + if (curr->opcode == _SET_IP) { + int32_t old_target = (int32_t)uop_get_target(curr); + curr++; + trace_length++; + curr->opcode = _EXIT_TRACE; + curr->format = UOP_FORMAT_TARGET; + curr->target = old_target; } - } - if (opcode == ENTER_EXECUTOR) { - // We have a couple of options here. We *could* peek "underneath" - // this executor and continue tracing, which could give us a longer, - // more optimizeable trace (at the expense of lots of duplicated - // tier two code). Instead, we choose to just end here and stitch to - // the other trace, which allows a side-exit traces to rejoin the - // "main" trace periodically (and also helps protect us against - // pathological behavior where the amount of tier two code explodes - // for a medium-length, branchy code path). This seems to work - // better in practice, but in the future we could be smarter about - // what we do here: goto done; } - assert(opcode != ENTER_EXECUTOR && opcode != EXTENDED_ARG); - RESERVE_RAW(2, "_CHECK_VALIDITY"); - ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target); - if (!OPCODE_HAS_NO_SAVE_IP(opcode)) { - RESERVE_RAW(2, "_SET_IP"); - ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)instr, target); - } + } - /* Special case the first instruction, - * so that we can guarantee forward progress */ - if (first && progress_needed) { - assert(first); - if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) { - opcode = _PyOpcode_Deopt[opcode]; - } - assert(!OPCODE_HAS_EXIT(opcode)); - assert(!OPCODE_HAS_DEOPT(opcode)); - } + if (opcode == NOP) { + return 1; + } - if (OPCODE_HAS_EXIT(opcode)) { - // Make space for side exit and final _EXIT_TRACE: - RESERVE_RAW(2, "_EXIT_TRACE"); - max_length--; - } - if (OPCODE_HAS_ERROR(opcode)) { - // Make space for error stub and final _EXIT_TRACE: - RESERVE_RAW(2, "_ERROR_POP_N"); - max_length--; - } - switch (opcode) { - case POP_JUMP_IF_NONE: - case POP_JUMP_IF_NOT_NONE: - case POP_JUMP_IF_FALSE: - case POP_JUMP_IF_TRUE: - { - RESERVE(1); - int counter = instr[1].cache; - int bitcount = _Py_popcount32(counter); - int jump_likely = bitcount > 8; - /* If bitcount is 8 (half the jumps were taken), adjust confidence by 50%. - For values in between, adjust proportionally. */ - if (jump_likely) { - confidence = confidence * bitcount / 16; - } - else { - confidence = confidence * (16 - bitcount) / 16; - } - uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_likely]; - DPRINTF(2, "%d: %s(%d): counter=%04x, bitcount=%d, likely=%d, confidence=%d, uopcode=%s\n", - target, _PyOpcode_OpName[opcode], oparg, - counter, bitcount, jump_likely, confidence, _PyUOpName(uopcode)); - if (confidence < CONFIDENCE_CUTOFF) { - DPRINTF(2, "Confidence too low (%d < %d)\n", confidence, CONFIDENCE_CUTOFF); - OPT_STAT_INC(low_confidence); - goto done; - } - _Py_CODEUNIT *next_instr = instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; - _Py_CODEUNIT *target_instr = next_instr + oparg; - if (jump_likely) { - DPRINTF(2, "Jump likely (%04x = %d bits), continue at byte offset %d\n", - instr[1].cache, bitcount, 2 * INSTR_IP(target_instr, code)); - instr = target_instr; - ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(next_instr, code)); - goto top; - } - ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(target_instr, code)); - break; - } + if (opcode == JUMP_FORWARD) { + return 1; + } - case JUMP_BACKWARD: - case JUMP_BACKWARD_JIT: - ADD_TO_TRACE(_CHECK_PERIODIC, 0, 0, target); - _Py_FALLTHROUGH; - case JUMP_BACKWARD_NO_INTERRUPT: - { - instr += 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] - (int)oparg; - if (jump_seen) { - OPT_STAT_INC(inner_loop); - DPRINTF(2, "JUMP_BACKWARD not to top ends trace\n"); - goto done; - } - jump_seen = true; - goto top; - } + if (opcode == EXTENDED_ARG) { + return 1; + } - case JUMP_FORWARD: - { - RESERVE(0); - // This will emit two _SET_IP instructions; leave it to the optimizer - instr += oparg; - break; - } + // One for possible _DEOPT, one because _CHECK_VALIDITY itself might _DEOPT + max_length -= 2; - case RESUME: - /* Use a special tier 2 version of RESUME_CHECK to allow traces to - * start with RESUME_CHECK */ - ADD_TO_TRACE(_TIER2_RESUME_CHECK, 0, 0, target); - break; + const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; - default: - { - const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; - if (expansion->nuops > 0) { - // Reserve space for nuops (+ _SET_IP + _EXIT_TRACE) - int nuops = expansion->nuops; - RESERVE(nuops + 1); /* One extra for exit */ - int16_t last_op = expansion->uops[nuops-1].uop; - if (last_op == _RETURN_VALUE || last_op == _RETURN_GENERATOR || last_op == _YIELD_VALUE) { - // Check for trace stack underflow now: - // We can't bail e.g. in the middle of - // LOAD_CONST + _RETURN_VALUE. - if (trace_stack_depth == 0) { - DPRINTF(2, "Trace stack underflow\n"); - OPT_STAT_INC(trace_stack_underflow); - return 0; - } - } - uint32_t orig_oparg = oparg; // For OPARG_TOP/BOTTOM - for (int i = 0; i < nuops; i++) { - oparg = orig_oparg; - uint32_t uop = expansion->uops[i].uop; - uint64_t operand = 0; - // Add one to account for the actual opcode/oparg pair: - int offset = expansion->uops[i].offset + 1; - switch (expansion->uops[i].size) { - case OPARG_SIMPLE: - assert(opcode != JUMP_BACKWARD_NO_INTERRUPT && opcode != JUMP_BACKWARD); - break; - case OPARG_CACHE_1: - operand = read_u16(&instr[offset].cache); - break; - case OPARG_CACHE_2: - operand = read_u32(&instr[offset].cache); - break; - case OPARG_CACHE_4: - operand = read_u64(&instr[offset].cache); - break; - case OPARG_TOP: // First half of super-instr - oparg = orig_oparg >> 4; - break; - case OPARG_BOTTOM: // Second half of super-instr - oparg = orig_oparg & 0xF; - break; - case OPARG_SAVE_RETURN_OFFSET: // op=_SAVE_RETURN_OFFSET; oparg=return_offset - oparg = offset; - assert(uop == _SAVE_RETURN_OFFSET); - break; - case OPARG_REPLACED: - uop = _PyUOp_Replacements[uop]; - assert(uop != 0); - uint32_t next_inst = target + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + (oparg > 255); - if (uop == _TIER2_RESUME_CHECK) { - target = next_inst; - } -#ifdef Py_DEBUG - else { - uint32_t jump_target = next_inst + oparg; - assert(_Py_GetBaseCodeUnit(code, jump_target).op.code == END_FOR); - assert(_Py_GetBaseCodeUnit(code, jump_target+1).op.code == POP_ITER); - } -#endif - break; - case OPERAND1_1: - assert(trace[trace_length-1].opcode == uop); - operand = read_u16(&instr[offset].cache); - trace[trace_length-1].operand1 = operand; - continue; - case OPERAND1_2: - assert(trace[trace_length-1].opcode == uop); - operand = read_u32(&instr[offset].cache); - trace[trace_length-1].operand1 = operand; - continue; - case OPERAND1_4: - assert(trace[trace_length-1].opcode == uop); - operand = read_u64(&instr[offset].cache); - trace[trace_length-1].operand1 = operand; - continue; - default: - fprintf(stderr, - "opcode=%d, oparg=%d; nuops=%d, i=%d; size=%d, offset=%d\n", - opcode, oparg, nuops, i, - expansion->uops[i].size, - expansion->uops[i].offset); - Py_FatalError("garbled expansion"); - } + assert(opcode != ENTER_EXECUTOR && opcode != EXTENDED_ARG); + assert(!_PyErr_Occurred(tstate)); - if (uop == _RETURN_VALUE || uop == _RETURN_GENERATOR || uop == _YIELD_VALUE) { - TRACE_STACK_POP(); - /* Set the operand to the function or code object returned to, - * to assist optimization passes. (See _PUSH_FRAME below.) - */ - if (func != NULL) { - operand = (uintptr_t)func; - } - else if (code != NULL) { - operand = (uintptr_t)code | 1; - } - else { - operand = 0; - } - ADD_TO_TRACE(uop, oparg, operand, target); - DPRINTF(2, - "Returning to %s (%s:%d) at byte offset %d\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - 2 * INSTR_IP(instr, code)); - goto top; - } - if (uop == _PUSH_FRAME) { - assert(i + 1 == nuops); - if (opcode == FOR_ITER_GEN || - opcode == LOAD_ATTR_PROPERTY || - opcode == BINARY_OP_SUBSCR_GETITEM || - opcode == SEND_GEN) - { - DPRINTF(2, "Bailing due to dynamic target\n"); - OPT_STAT_INC(unknown_callee); - return 0; - } - assert(_PyOpcode_Deopt[opcode] == CALL || _PyOpcode_Deopt[opcode] == CALL_KW); - int func_version_offset = - offsetof(_PyCallCache, func_version)/sizeof(_Py_CODEUNIT) - // Add one to account for the actual opcode/oparg pair: - + 1; - uint32_t func_version = read_u32(&instr[func_version_offset].cache); - PyCodeObject *new_code = NULL; - PyFunctionObject *new_func = - _PyFunction_LookupByVersion(func_version, (PyObject **) &new_code); - DPRINTF(2, "Function: version=%#x; new_func=%p, new_code=%p\n", - (int)func_version, new_func, new_code); - if (new_code != NULL) { - if (new_code == code) { - // Recursive call, bail (we could be here forever). - DPRINTF(2, "Bailing on recursive call to %s (%s:%d)\n", - PyUnicode_AsUTF8(new_code->co_qualname), - PyUnicode_AsUTF8(new_code->co_filename), - new_code->co_firstlineno); - OPT_STAT_INC(recursive_call); - ADD_TO_TRACE(uop, oparg, 0, target); - ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); - goto done; - } - if (new_code->co_version != func_version) { - // func.__code__ was updated. - // Perhaps it may happen again, so don't bother tracing. - // TODO: Reason about this -- is it better to bail or not? - DPRINTF(2, "Bailing because co_version != func_version\n"); - ADD_TO_TRACE(uop, oparg, 0, target); - ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); - goto done; - } - // Increment IP to the return address - instr += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + 1; - TRACE_STACK_PUSH(); - _Py_BloomFilter_Add(dependencies, new_code); - /* Set the operand to the callee's function or code object, - * to assist optimization passes. - * We prefer setting it to the function - * but if that's not available but the code is available, - * use the code, setting the low bit so the optimizer knows. - */ - if (new_func != NULL) { - operand = (uintptr_t)new_func; - } - else if (new_code != NULL) { - operand = (uintptr_t)new_code | 1; - } - else { - operand = 0; - } - ADD_TO_TRACE(uop, oparg, operand, target); - code = new_code; - func = new_func; - instr = _PyCode_CODE(code); - DPRINTF(2, - "Continuing in %s (%s:%d) at byte offset %d\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - 2 * INSTR_IP(instr, code)); - goto top; + if (OPCODE_HAS_EXIT(opcode)) { + // Make space for side exit and final _EXIT_TRACE: + max_length--; + } + if (OPCODE_HAS_ERROR(opcode)) { + // Make space for error stub and final _EXIT_TRACE: + max_length--; + } + + // _GUARD_IP leads to an exit. + max_length -= needs_guard_ip; + + RESERVE_RAW(expansion->nuops + needs_guard_ip + 2 + (!OPCODE_HAS_NO_SAVE_IP(opcode)), "uop and various checks"); + + ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target); + + if (!OPCODE_HAS_NO_SAVE_IP(opcode)) { + ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)target_instr, target); + } + + // Can be NULL for the entry frame. + if (old_code != NULL) { + _Py_BloomFilter_Add(dependencies, old_code); + } + + switch (opcode) { + case POP_JUMP_IF_NONE: + case POP_JUMP_IF_NOT_NONE: + case POP_JUMP_IF_FALSE: + case POP_JUMP_IF_TRUE: + { + _Py_CODEUNIT *computed_next_instr_without_modifiers = target_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; + _Py_CODEUNIT *computed_next_instr = computed_next_instr_without_modifiers + (computed_next_instr_without_modifiers->op.code == NOT_TAKEN); + _Py_CODEUNIT *computed_jump_instr = computed_next_instr_without_modifiers + oparg; + assert(next_instr == computed_next_instr || next_instr == computed_jump_instr); + int jump_happened = computed_jump_instr == next_instr; + assert(jump_happened == (target_instr[1].cache & 1)); + uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_happened]; + ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(jump_happened ? computed_next_instr : computed_jump_instr, old_code)); + break; + } + case JUMP_BACKWARD_JIT: + // This is possible as the JIT might have re-activated after it was disabled + case JUMP_BACKWARD_NO_JIT: + case JUMP_BACKWARD: + ADD_TO_TRACE(_CHECK_PERIODIC, 0, 0, target); + _Py_FALLTHROUGH; + case JUMP_BACKWARD_NO_INTERRUPT: + { + if ((next_instr != _tstate->jit_tracer_state.initial_state.close_loop_instr) && + (next_instr != _tstate->jit_tracer_state.initial_state.start_instr) && + _tstate->jit_tracer_state.prev_state.code_curr_size > CODE_SIZE_NO_PROGRESS && + // For side exits, we don't want to terminate them early. + _tstate->jit_tracer_state.initial_state.exit == NULL && + // These are coroutines, and we want to unroll those usually. + opcode != JUMP_BACKWARD_NO_INTERRUPT) { + // We encountered a JUMP_BACKWARD but not to the top of our own loop. + // We don't want to continue tracing as we might get stuck in the + // inner loop. Instead, end the trace where the executor of the + // inner loop might start and let the traces rejoin. + OPT_STAT_INC(inner_loop); + ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); + trace[trace_length-1].operand1 = true; // is_control_flow + DPRINTF(2, "JUMP_BACKWARD not to top ends trace %p %p %p\n", next_instr, + _tstate->jit_tracer_state.initial_state.close_loop_instr, _tstate->jit_tracer_state.initial_state.start_instr); + goto done; + } + break; + } + + case RESUME: + case RESUME_CHECK: + /* Use a special tier 2 version of RESUME_CHECK to allow traces to + * start with RESUME_CHECK */ + ADD_TO_TRACE(_TIER2_RESUME_CHECK, 0, 0, target); + break; + default: + { + const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; + // Reserve space for nuops (+ _SET_IP + _EXIT_TRACE) + int nuops = expansion->nuops; + if (nuops == 0) { + DPRINTF(2, "Unsupported opcode %s\n", _PyOpcode_OpName[opcode]); + goto unsupported; + } + assert(nuops > 0); + uint32_t orig_oparg = oparg; // For OPARG_TOP/BOTTOM + uint32_t orig_target = target; + for (int i = 0; i < nuops; i++) { + oparg = orig_oparg; + target = orig_target; + uint32_t uop = expansion->uops[i].uop; + uint64_t operand = 0; + // Add one to account for the actual opcode/oparg pair: + int offset = expansion->uops[i].offset + 1; + switch (expansion->uops[i].size) { + case OPARG_SIMPLE: + assert(opcode != _JUMP_BACKWARD_NO_INTERRUPT && opcode != JUMP_BACKWARD); + break; + case OPARG_CACHE_1: + operand = read_u16(&this_instr[offset].cache); + break; + case OPARG_CACHE_2: + operand = read_u32(&this_instr[offset].cache); + break; + case OPARG_CACHE_4: + operand = read_u64(&this_instr[offset].cache); + break; + case OPARG_TOP: // First half of super-instr + assert(orig_oparg <= 255); + oparg = orig_oparg >> 4; + break; + case OPARG_BOTTOM: // Second half of super-instr + assert(orig_oparg <= 255); + oparg = orig_oparg & 0xF; + break; + case OPARG_SAVE_RETURN_OFFSET: // op=_SAVE_RETURN_OFFSET; oparg=return_offset + oparg = offset; + assert(uop == _SAVE_RETURN_OFFSET); + break; + case OPARG_REPLACED: + uop = _PyUOp_Replacements[uop]; + assert(uop != 0); + + uint32_t next_inst = target + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; + if (uop == _TIER2_RESUME_CHECK) { + target = next_inst; + } + else { + int extended_arg = orig_oparg > 255; + uint32_t jump_target = next_inst + orig_oparg + extended_arg; + assert(_Py_GetBaseCodeUnit(old_code, jump_target).op.code == END_FOR); + assert(_Py_GetBaseCodeUnit(old_code, jump_target+1).op.code == POP_ITER); + if (is_for_iter_test[uop]) { + target = jump_target + 1; } - DPRINTF(2, "Bail, new_code == NULL\n"); - OPT_STAT_INC(unknown_callee); - return 0; } - - if (uop == _BINARY_OP_INPLACE_ADD_UNICODE) { - assert(i + 1 == nuops); - _Py_CODEUNIT *next_instr = instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; - assert(next_instr->op.code == STORE_FAST); - operand = next_instr->op.arg; - // Skip the STORE_FAST: - instr++; + break; + case OPERAND1_1: + assert(trace[trace_length-1].opcode == uop); + operand = read_u16(&this_instr[offset].cache); + trace[trace_length-1].operand1 = operand; + continue; + case OPERAND1_2: + assert(trace[trace_length-1].opcode == uop); + operand = read_u32(&this_instr[offset].cache); + trace[trace_length-1].operand1 = operand; + continue; + case OPERAND1_4: + assert(trace[trace_length-1].opcode == uop); + operand = read_u64(&this_instr[offset].cache); + trace[trace_length-1].operand1 = operand; + continue; + default: + fprintf(stderr, + "opcode=%d, oparg=%d; nuops=%d, i=%d; size=%d, offset=%d\n", + opcode, oparg, nuops, i, + expansion->uops[i].size, + expansion->uops[i].offset); + Py_FatalError("garbled expansion"); + } + if (uop == _PUSH_FRAME || uop == _RETURN_VALUE || uop == _RETURN_GENERATOR || uop == _YIELD_VALUE) { + PyCodeObject *new_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + PyFunctionObject *new_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + + operand = 0; + if (frame->owner < FRAME_OWNED_BY_INTERPRETER) { + // Don't add nested code objects to the dependency. + // It causes endless re-traces. + if (new_func != NULL && !Py_IsNone((PyObject*)new_func) && !(new_code->co_flags & CO_NESTED)) { + operand = (uintptr_t)new_func; + DPRINTF(2, "Adding %p func to op\n", (void *)operand); + _Py_BloomFilter_Add(dependencies, new_func); + } + else if (new_code != NULL && !Py_IsNone((PyObject*)new_code)) { + operand = (uintptr_t)new_code | 1; + DPRINTF(2, "Adding %p code to op\n", (void *)operand); + _Py_BloomFilter_Add(dependencies, new_code); } - - // All other instructions - ADD_TO_TRACE(uop, oparg, operand, target); } + ADD_TO_TRACE(uop, oparg, operand, target); + trace[trace_length - 1].operand1 = PyStackRef_IsNone(frame->f_executable) ? 2 : ((int)(frame->stackpointer - _PyFrame_Stackbase(frame))); break; } - DPRINTF(2, "Unsupported opcode %s\n", _PyOpcode_OpName[opcode]); - OPT_UNSUPPORTED_OPCODE(opcode); - goto done; // Break out of loop - } // End default - - } // End switch (opcode) + if (uop == _BINARY_OP_INPLACE_ADD_UNICODE) { + assert(i + 1 == nuops); + _Py_CODEUNIT *next = target_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; + assert(next->op.code == STORE_FAST); + operand = next->op.arg; + } + // All other instructions + ADD_TO_TRACE(uop, oparg, operand, target); + } + break; + } // End default - instr++; - // Add cache size for opcode - instr += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; + } // End switch (opcode) - if (opcode == CALL_LIST_APPEND) { - assert(instr->op.code == POP_TOP); - instr++; + if (needs_guard_ip) { + uint16_t guard_ip = guard_ip_uop[trace[trace_length-1].opcode]; + if (guard_ip == 0) { + DPRINTF(1, "Unknown uop needing guard ip %s\n", _PyOpcode_uop_name[trace[trace_length-1].opcode]); + Py_UNREACHABLE(); } - top: - // Jump here after _PUSH_FRAME or likely branches. - first = false; - } // End for (;;) - + ADD_TO_TRACE(guard_ip, 0, (uintptr_t)next_instr, 0); + } + // Loop back to the start + int is_first_instr = _tstate->jit_tracer_state.initial_state.close_loop_instr == next_instr || + _tstate->jit_tracer_state.initial_state.start_instr == next_instr; + if (is_first_instr && _tstate->jit_tracer_state.prev_state.code_curr_size > CODE_SIZE_NO_PROGRESS) { + if (needs_guard_ip) { + ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)next_instr, 0); + } + ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0); + goto done; + } + DPRINTF(2, "Trace continuing\n"); + _tstate->jit_tracer_state.prev_state.code_curr_size = trace_length; + _tstate->jit_tracer_state.prev_state.code_max_size = max_length; + return 1; done: - while (trace_stack_depth > 0) { - TRACE_STACK_POP(); - } - assert(code == initial_code); - // Skip short traces where we can't even translate a single instruction: - if (first) { - OPT_STAT_INC(trace_too_short); - DPRINTF(2, - "No trace for %s (%s:%d) at byte offset %d (no progress)\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - 2 * INSTR_IP(initial_instr, code)); + DPRINTF(2, "Trace done\n"); + _tstate->jit_tracer_state.prev_state.code_curr_size = trace_length; + _tstate->jit_tracer_state.prev_state.code_max_size = max_length; + return 0; +full: + DPRINTF(2, "Trace full\n"); + if (!is_terminator(&_tstate->jit_tracer_state.code_buffer[trace_length-1])) { + // Undo the last few instructions. + trace_length = _tstate->jit_tracer_state.prev_state.code_curr_size; + max_length = _tstate->jit_tracer_state.prev_state.code_max_size; + // We previously reversed one. + max_length += 1; + ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); + trace[trace_length-1].operand1 = true; // is_control_flow + } + _tstate->jit_tracer_state.prev_state.code_curr_size = trace_length; + _tstate->jit_tracer_state.prev_state.code_max_size = max_length; + return 0; +} + +// Returns 0 for do not enter tracing, 1 on enter tracing. +int +_PyJit_TryInitializeTracing( + PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *curr_instr, + _Py_CODEUNIT *start_instr, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, + _PyExitData *exit, int oparg) +{ + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + // A recursive trace. + // Don't trace into the inner call because it will stomp on the previous trace, causing endless retraces. + if (_tstate->jit_tracer_state.prev_state.code_curr_size > CODE_SIZE_EMPTY) { return 0; } - if (!is_terminator(&trace[trace_length-1])) { - /* Allow space for _EXIT_TRACE */ - max_length += 2; - ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); + if (oparg > 0xFFFF) { + return 0; + } + if (_tstate->jit_tracer_state.code_buffer == NULL) { + _tstate->jit_tracer_state.code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); + if (_tstate->jit_tracer_state.code_buffer == NULL) { + // Don't error, just go to next instruction. + return 0; + } + } + PyObject *func = PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + if (func == NULL) { + return 0; + } + PyCodeObject *code = _PyFrame_GetCode(frame); +#ifdef Py_DEBUG + char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); + int lltrace = 0; + if (python_lltrace != NULL && *python_lltrace >= '0') { + lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that } - DPRINTF(1, - "Created a proto-trace for %s (%s:%d) at byte offset %d -- length %d\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - 2 * INSTR_IP(initial_instr, code), - trace_length); - OPT_HIST(trace_length, trace_length_hist); - return trace_length; + DPRINTF(2, + "Tracing %s (%s:%d) at byte offset %d at chain depth %d\n", + PyUnicode_AsUTF8(code->co_qualname), + PyUnicode_AsUTF8(code->co_filename), + code->co_firstlineno, + 2 * INSTR_IP(close_loop_instr, code), + chain_depth); +#endif + + add_to_trace(_tstate->jit_tracer_state.code_buffer, 0, _START_EXECUTOR, 0, (uintptr_t)start_instr, INSTR_IP(start_instr, code)); + add_to_trace(_tstate->jit_tracer_state.code_buffer, 1, _MAKE_WARM, 0, 0, 0); + _tstate->jit_tracer_state.prev_state.code_curr_size = CODE_SIZE_EMPTY; + + _tstate->jit_tracer_state.prev_state.code_max_size = UOP_MAX_TRACE_LENGTH; + _tstate->jit_tracer_state.initial_state.start_instr = start_instr; + _tstate->jit_tracer_state.initial_state.close_loop_instr = close_loop_instr; + _tstate->jit_tracer_state.initial_state.code = (PyCodeObject *)Py_NewRef(code); + _tstate->jit_tracer_state.initial_state.func = (PyFunctionObject *)Py_NewRef(func); + _tstate->jit_tracer_state.initial_state.exit = exit; + _tstate->jit_tracer_state.initial_state.stack_depth = curr_stackdepth; + _tstate->jit_tracer_state.initial_state.chain_depth = chain_depth; + _tstate->jit_tracer_state.prev_state.instr_frame = frame; + _tstate->jit_tracer_state.prev_state.dependencies_still_valid = true; + _tstate->jit_tracer_state.prev_state.instr_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame)); + _tstate->jit_tracer_state.prev_state.instr = curr_instr; + _tstate->jit_tracer_state.prev_state.instr_frame = frame; + _tstate->jit_tracer_state.prev_state.instr_oparg = oparg; + _tstate->jit_tracer_state.prev_state.instr_stacklevel = curr_stackdepth; + _tstate->jit_tracer_state.prev_state.instr_is_super = false; + assert(curr_instr->op.code == JUMP_BACKWARD_JIT || (exit != NULL)); + _tstate->jit_tracer_state.initial_state.jump_backward_instr = curr_instr; + + if (_PyOpcode_Caches[_PyOpcode_Deopt[close_loop_instr->op.code]]) { + close_loop_instr[1].counter = trigger_backoff_counter(); + } + _Py_BloomFilter_Init(&_tstate->jit_tracer_state.prev_state.dependencies); + return 1; +} + +void +_PyJit_FinalizeTracing(PyThreadState *tstate) +{ + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + Py_CLEAR(_tstate->jit_tracer_state.initial_state.code); + Py_CLEAR(_tstate->jit_tracer_state.initial_state.func); + Py_CLEAR(_tstate->jit_tracer_state.prev_state.instr_code); + _tstate->jit_tracer_state.prev_state.code_curr_size = CODE_SIZE_EMPTY; + _tstate->jit_tracer_state.prev_state.code_max_size = UOP_MAX_TRACE_LENGTH - 1; } + #undef RESERVE #undef RESERVE_RAW #undef INSTR_IP @@ -1018,20 +1064,21 @@ count_exits(_PyUOpInstruction *buffer, int length) int exit_count = 0; for (int i = 0; i < length; i++) { int opcode = buffer[i].opcode; - if (opcode == _EXIT_TRACE) { + if (opcode == _EXIT_TRACE || opcode == _DYNAMIC_EXIT) { exit_count++; } } return exit_count; } -static void make_exit(_PyUOpInstruction *inst, int opcode, int target) +static void make_exit(_PyUOpInstruction *inst, int opcode, int target, bool is_control_flow) { inst->opcode = opcode; inst->oparg = 0; inst->operand0 = 0; inst->format = UOP_FORMAT_TARGET; inst->target = target; + inst->operand1 = is_control_flow; #ifdef Py_STATS inst->execution_count = 0; #endif @@ -1075,15 +1122,17 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) exit_op = _HANDLE_PENDING_AND_DEOPT; } int32_t jump_target = target; - if (is_for_iter_test[opcode]) { - /* Target the POP_TOP immediately after the END_FOR, - * leaving only the iterator on the stack. */ - int extended_arg = inst->oparg > 255; - int32_t next_inst = target + 1 + INLINE_CACHE_ENTRIES_FOR_ITER + extended_arg; - jump_target = next_inst + inst->oparg + 1; + if ( + opcode == _GUARD_IP__PUSH_FRAME || + opcode == _GUARD_IP_RETURN_VALUE || + opcode == _GUARD_IP_YIELD_VALUE || + opcode == _GUARD_IP_RETURN_GENERATOR + ) { + exit_op = _DYNAMIC_EXIT; } + bool is_control_flow = (opcode == _GUARD_IS_FALSE_POP || opcode == _GUARD_IS_TRUE_POP || is_for_iter_test[opcode]); if (jump_target != current_jump_target || current_exit_op != exit_op) { - make_exit(&buffer[next_spare], exit_op, jump_target); + make_exit(&buffer[next_spare], exit_op, jump_target, is_control_flow); current_exit_op = exit_op; current_jump_target = jump_target; current_jump = next_spare; @@ -1099,7 +1148,7 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) current_popped = popped; current_error = next_spare; current_error_target = target; - make_exit(&buffer[next_spare], _ERROR_POP_N, 0); + make_exit(&buffer[next_spare], _ERROR_POP_N, 0, false); buffer[next_spare].operand0 = target; next_spare++; } @@ -1157,7 +1206,9 @@ sanity_check(_PyExecutorObject *executor) } bool ended = false; uint32_t i = 0; - CHECK(executor->trace[0].opcode == _START_EXECUTOR || executor->trace[0].opcode == _COLD_EXIT); + CHECK(executor->trace[0].opcode == _START_EXECUTOR || + executor->trace[0].opcode == _COLD_EXIT || + executor->trace[0].opcode == _COLD_DYNAMIC_EXIT); for (; i < executor->code_size; i++) { const _PyUOpInstruction *inst = &executor->trace[i]; uint16_t opcode = inst->opcode; @@ -1189,7 +1240,8 @@ sanity_check(_PyExecutorObject *executor) opcode == _DEOPT || opcode == _HANDLE_PENDING_AND_DEOPT || opcode == _EXIT_TRACE || - opcode == _ERROR_POP_N); + opcode == _ERROR_POP_N || + opcode == _DYNAMIC_EXIT); } } @@ -1202,7 +1254,7 @@ sanity_check(_PyExecutorObject *executor) * and not a NOP. */ static _PyExecutorObject * -make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies) +make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies, int chain_depth) { int exit_count = count_exits(buffer, length); _PyExecutorObject *executor = allocate_executor(exit_count, length); @@ -1212,10 +1264,11 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil /* Initialize exits */ _PyExecutorObject *cold = _PyExecutor_GetColdExecutor(); + _PyExecutorObject *cold_dynamic = _PyExecutor_GetColdDynamicExecutor(); + cold->vm_data.chain_depth = chain_depth; for (int i = 0; i < exit_count; i++) { executor->exits[i].index = i; executor->exits[i].temperature = initial_temperature_backoff_counter(); - executor->exits[i].executor = cold; } int next_exit = exit_count-1; _PyUOpInstruction *dest = (_PyUOpInstruction *)&executor->trace[length]; @@ -1225,11 +1278,13 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil int opcode = buffer[i].opcode; dest--; *dest = buffer[i]; - assert(opcode != _POP_JUMP_IF_FALSE && opcode != _POP_JUMP_IF_TRUE); - if (opcode == _EXIT_TRACE) { + if (opcode == _EXIT_TRACE || opcode == _DYNAMIC_EXIT) { _PyExitData *exit = &executor->exits[next_exit]; exit->target = buffer[i].target; dest->operand0 = (uint64_t)exit; + exit->executor = opcode == _EXIT_TRACE ? cold : cold_dynamic; + exit->is_dynamic = (char)(opcode == _DYNAMIC_EXIT); + exit->is_control_flow = (char)buffer[i].operand1; next_exit--; } } @@ -1291,38 +1346,32 @@ int effective_trace_length(_PyUOpInstruction *buffer, int length) static int uop_optimize( _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr, + PyThreadState *tstate, _PyExecutorObject **exec_ptr, - int curr_stackentries, bool progress_needed) { - _PyBloomFilter dependencies; - _Py_BloomFilter_Init(&dependencies); - PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->jit_uop_buffer == NULL) { - interp->jit_uop_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); - if (interp->jit_uop_buffer == NULL) { - return 0; - } - } - _PyUOpInstruction *buffer = interp->jit_uop_buffer; + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + _PyBloomFilter *dependencies = &_tstate->jit_tracer_state.prev_state.dependencies; + _PyUOpInstruction *buffer = _tstate->jit_tracer_state.code_buffer; OPT_STAT_INC(attempts); char *env_var = Py_GETENV("PYTHON_UOPS_OPTIMIZE"); bool is_noopt = true; if (env_var == NULL || *env_var == '\0' || *env_var > '0') { is_noopt = false; } - int length = translate_bytecode_to_trace(frame, instr, buffer, UOP_MAX_TRACE_LENGTH, &dependencies, progress_needed); - if (length <= 0) { - // Error or nothing translated - return length; + int curr_stackentries = _tstate->jit_tracer_state.initial_state.stack_depth; + int length = _tstate->jit_tracer_state.prev_state.code_curr_size; + if (length <= CODE_SIZE_NO_PROGRESS) { + return 0; } + assert(length > 0); assert(length < UOP_MAX_TRACE_LENGTH); OPT_STAT_INC(traces_created); if (!is_noopt) { - length = _Py_uop_analyze_and_optimize(frame, buffer, - length, - curr_stackentries, &dependencies); + length = _Py_uop_analyze_and_optimize( + _tstate->jit_tracer_state.initial_state.func, + buffer,length, + curr_stackentries, dependencies); if (length <= 0) { return length; } @@ -1345,14 +1394,14 @@ uop_optimize( OPT_HIST(effective_trace_length(buffer, length), optimized_trace_length_hist); length = prepare_for_execution(buffer, length); assert(length <= UOP_MAX_TRACE_LENGTH); - _PyExecutorObject *executor = make_executor_from_uops(buffer, length, &dependencies); + _PyExecutorObject *executor = make_executor_from_uops( + buffer, length, dependencies, _tstate->jit_tracer_state.initial_state.chain_depth); if (executor == NULL) { return -1; } assert(length <= UOP_MAX_TRACE_LENGTH); // Check executor coldness - PyThreadState *tstate = PyThreadState_Get(); // It's okay if this ends up going negative. if (--tstate->interp->executor_creation_counter == 0) { _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT); @@ -1539,6 +1588,35 @@ _PyExecutor_GetColdExecutor(void) return cold; } +_PyExecutorObject * +_PyExecutor_GetColdDynamicExecutor(void) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (interp->cold_dynamic_executor != NULL) { + assert(interp->cold_dynamic_executor->trace[0].opcode == _COLD_DYNAMIC_EXIT); + return interp->cold_dynamic_executor; + } + _PyExecutorObject *cold = allocate_executor(0, 1); + if (cold == NULL) { + Py_FatalError("Cannot allocate core JIT code"); + } + ((_PyUOpInstruction *)cold->trace)->opcode = _COLD_DYNAMIC_EXIT; +#ifdef _Py_JIT + cold->jit_code = NULL; + cold->jit_size = 0; + // This is initialized to true so we can prevent the executor + // from being immediately detected as cold and invalidated. + cold->vm_data.warm = true; + if (_PyJIT_Compile(cold, cold->trace, 1)) { + Py_DECREF(cold); + Py_FatalError("Cannot allocate core JIT code"); + } +#endif + _Py_SetImmortal((PyObject *)cold); + interp->cold_dynamic_executor = cold; + return cold; +} + void _PyExecutor_ClearExit(_PyExitData *exit) { @@ -1546,7 +1624,12 @@ _PyExecutor_ClearExit(_PyExitData *exit) return; } _PyExecutorObject *old = exit->executor; - exit->executor = _PyExecutor_GetColdExecutor(); + if (exit->is_dynamic) { + exit->executor = _PyExecutor_GetColdDynamicExecutor(); + } + else { + exit->executor = _PyExecutor_GetColdExecutor(); + } Py_DECREF(old); } @@ -1648,6 +1731,18 @@ _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int is _Py_Executors_InvalidateAll(interp, is_invalidation); } +void +_PyJit_Tracer_InvalidateDependency(PyThreadState *tstate, void *obj) +{ + _PyBloomFilter obj_filter; + _Py_BloomFilter_Init(&obj_filter); + _Py_BloomFilter_Add(&obj_filter, obj); + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + if (bloom_filter_may_contain(&_tstate->jit_tracer_state.prev_state.dependencies, &obj_filter)) + { + _tstate->jit_tracer_state.prev_state.dependencies_still_valid = false; + } +} /* Invalidate all executors */ void _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation) @@ -1777,7 +1872,7 @@ executor_to_gv(_PyExecutorObject *executor, FILE *out) #ifdef Py_STATS fprintf(out, " %s -- %" PRIu64 "\n", i, opname, inst->execution_count); #else - fprintf(out, " %s\n", i, opname); + fprintf(out, " %s op0=%" PRIu64 "\n", i, opname, inst->operand0); #endif if (inst->opcode == _EXIT_TRACE || inst->opcode == _JUMP_TO_TOP) { break; @@ -1787,6 +1882,8 @@ executor_to_gv(_PyExecutorObject *executor, FILE *out) fprintf(out, "]\n\n"); /* Write all the outgoing edges */ + _PyExecutorObject *cold = _PyExecutor_GetColdExecutor(); + _PyExecutorObject *cold_dynamic = _PyExecutor_GetColdDynamicExecutor(); for (uint32_t i = 0; i < executor->code_size; i++) { _PyUOpInstruction const *inst = &executor->trace[i]; uint16_t flags = _PyUop_Flags[inst->opcode]; @@ -1797,10 +1894,10 @@ executor_to_gv(_PyExecutorObject *executor, FILE *out) else if (flags & HAS_EXIT_FLAG) { assert(inst->format == UOP_FORMAT_JUMP); _PyUOpInstruction const *exit_inst = &executor->trace[inst->jump_target]; - assert(exit_inst->opcode == _EXIT_TRACE); + assert(exit_inst->opcode == _EXIT_TRACE || exit_inst->opcode == _DYNAMIC_EXIT); exit = (_PyExitData *)exit_inst->operand0; } - if (exit != NULL && exit->executor != NULL) { + if (exit != NULL && exit->executor != cold && exit->executor != cold_dynamic) { fprintf(out, "executor_%p:i%d -> executor_%p:start\n", executor, i, exit->executor); } if (inst->opcode == _EXIT_TRACE || inst->opcode == _JUMP_TO_TOP) { diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index a6add301ccb26c..8d7b734e17cb0b 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -142,8 +142,10 @@ incorrect_keys(PyObject *obj, uint32_t version) #define STACK_LEVEL() ((int)(stack_pointer - ctx->frame->stack)) #define STACK_SIZE() ((int)(ctx->frame->stack_len)) +#define CURRENT_FRAME_IS_INIT_SHIM() (ctx->frame->code == ((PyCodeObject *)&_Py_InitCleanup)) + #define WITHIN_STACK_BOUNDS() \ - (STACK_LEVEL() >= 0 && STACK_LEVEL() <= STACK_SIZE()) + (CURRENT_FRAME_IS_INIT_SHIM() || (STACK_LEVEL() >= 0 && STACK_LEVEL() <= STACK_SIZE())) #define GETLOCAL(idx) ((ctx->frame->locals[idx])) @@ -267,7 +269,7 @@ static PyCodeObject * get_current_code_object(JitOptContext *ctx) { - return (PyCodeObject *)ctx->frame->func->func_code; + return (PyCodeObject *)ctx->frame->code; } static PyObject * @@ -298,10 +300,6 @@ optimize_uops( JitOptContext context; JitOptContext *ctx = &context; uint32_t opcode = UINT16_MAX; - int curr_space = 0; - int max_space = 0; - _PyUOpInstruction *first_valid_check_stack = NULL; - _PyUOpInstruction *corresponding_check_stack = NULL; // Make sure that watchers are set up PyInterpreterState *interp = _PyInterpreterState_GET(); @@ -320,13 +318,18 @@ optimize_uops( ctx->frame = frame; _PyUOpInstruction *this_instr = NULL; + JitOptRef *stack_pointer = ctx->frame->stack_pointer; + for (int i = 0; !ctx->done; i++) { assert(i < trace_len); this_instr = &trace[i]; int oparg = this_instr->oparg; opcode = this_instr->opcode; - JitOptRef *stack_pointer = ctx->frame->stack_pointer; + + if (!CURRENT_FRAME_IS_INIT_SHIM()) { + stack_pointer = ctx->frame->stack_pointer; + } #ifdef Py_DEBUG if (get_lltrace() >= 3) { @@ -345,9 +348,11 @@ optimize_uops( Py_UNREACHABLE(); } assert(ctx->frame != NULL); - DPRINTF(3, " stack_level %d\n", STACK_LEVEL()); - ctx->frame->stack_pointer = stack_pointer; - assert(STACK_LEVEL() >= 0); + if (!CURRENT_FRAME_IS_INIT_SHIM()) { + DPRINTF(3, " stack_level %d\n", STACK_LEVEL()); + ctx->frame->stack_pointer = stack_pointer; + assert(STACK_LEVEL() >= 0); + } } if (ctx->out_of_space) { DPRINTF(3, "\n"); @@ -355,27 +360,21 @@ optimize_uops( } if (ctx->contradiction) { // Attempted to push a "bottom" (contradiction) symbol onto the stack. - // This means that the abstract interpreter has hit unreachable code. + // This means that the abstract interpreter has optimized to trace + // to an unreachable estate. // We *could* generate an _EXIT_TRACE or _FATAL_ERROR here, but hitting - // bottom indicates type instability, so we are probably better off + // bottom usually indicates an optimizer bug, so we are probably better off // retrying later. DPRINTF(3, "\n"); DPRINTF(1, "Hit bottom in abstract interpreter\n"); _Py_uop_abstractcontext_fini(ctx); + OPT_STAT_INC(optimizer_contradiction); return 0; } /* Either reached the end or cannot optimize further, but there * would be no benefit in retrying later */ _Py_uop_abstractcontext_fini(ctx); - if (first_valid_check_stack != NULL) { - assert(first_valid_check_stack->opcode == _CHECK_STACK_SPACE); - assert(max_space > 0); - assert(max_space <= INT_MAX); - assert(max_space <= INT32_MAX); - first_valid_check_stack->opcode = _CHECK_STACK_SPACE_OPERAND; - first_valid_check_stack->operand0 = max_space; - } return trace_len; error: @@ -460,6 +459,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) buffer[pc].opcode = _NOP; } break; + case _EXIT_TRACE: default: { // Cancel out pushes and pops, repeatedly. So: @@ -493,7 +493,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) } /* _PUSH_FRAME doesn't escape or error, but it * does need the IP for the return address */ - bool needs_ip = opcode == _PUSH_FRAME; + bool needs_ip = (opcode == _PUSH_FRAME || opcode == _YIELD_VALUE || opcode == _DYNAMIC_EXIT || opcode == _EXIT_TRACE); if (_PyUop_Flags[opcode] & HAS_ESCAPES_FLAG) { needs_ip = true; may_have_escaped = true; @@ -503,10 +503,14 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) buffer[last_set_ip].opcode = _SET_IP; last_set_ip = -1; } + if (opcode == _EXIT_TRACE) { + return pc + 1; + } break; } case _JUMP_TO_TOP: - case _EXIT_TRACE: + case _DYNAMIC_EXIT: + case _DEOPT: return pc + 1; } } @@ -518,7 +522,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) // > 0 - length of optimized trace int _Py_uop_analyze_and_optimize( - _PyInterpreterFrame *frame, + PyFunctionObject *func, _PyUOpInstruction *buffer, int length, int curr_stacklen, @@ -528,8 +532,8 @@ _Py_uop_analyze_and_optimize( OPT_STAT_INC(optimizer_attempts); length = optimize_uops( - _PyFrame_GetFunction(frame), buffer, - length, curr_stacklen, dependencies); + func, buffer, + length, curr_stacklen, dependencies); if (length == 0) { return length; diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index da3d3c96bc1d97..06fa8a4522a499 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -342,7 +342,6 @@ dummy_func(void) { int already_bool = optimize_to_bool(this_instr, ctx, value, &value); if (!already_bool) { sym_set_type(value, &PyBool_Type); - value = sym_new_truthiness(ctx, value, true); } } @@ -752,8 +751,14 @@ dummy_func(void) { } op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame)) { - new_frame = PyJitRef_NULL; - ctx->done = true; + assert((this_instr + 2)->opcode == _PUSH_FRAME); + PyCodeObject *co = get_code_with_logging((this_instr + 2)); + if (co == NULL) { + ctx->done = true; + break; + } + + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); } op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { @@ -764,8 +769,20 @@ dummy_func(void) { } op(_CREATE_INIT_FRAME, (init, self, args[oparg] -- init_frame)) { - init_frame = PyJitRef_NULL; - ctx->done = true; + ctx->frame->stack_pointer = stack_pointer - oparg - 2; + _Py_UOpsAbstractFrame *shim = frame_new(ctx, (PyCodeObject *)&_Py_InitCleanup, 0, NULL, 0); + if (shim == NULL) { + break; + } + /* Push self onto stack of shim */ + shim->stack[0] = self; + shim->stack_pointer++; + assert((int)(shim->stack_pointer - shim->stack) == 1); + ctx->frame = shim; + ctx->curr_frame_depth++; + assert((this_instr + 1)->opcode == _PUSH_FRAME); + PyCodeObject *co = get_code_with_logging((this_instr + 1)); + init_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, args-1, oparg+1)); } op(_RETURN_VALUE, (retval -- res)) { @@ -773,42 +790,65 @@ dummy_func(void) { JitOptRef temp = PyJitRef_StripReferenceInfo(retval); DEAD(retval); SAVE_STACK(); - PyCodeObject *co = get_current_code_object(ctx); ctx->frame->stack_pointer = stack_pointer; - frame_pop(ctx); + PyCodeObject *returning_code = get_code_with_logging(this_instr); + if (returning_code == NULL) { + ctx->done = true; + break; + } + int returning_stacklevel = this_instr->operand1; + if (ctx->curr_frame_depth >= 2) { + PyCodeObject *expected_code = ctx->frames[ctx->curr_frame_depth - 2].code; + if (expected_code == returning_code) { + assert((this_instr + 1)->opcode == _GUARD_IP_RETURN_VALUE); + REPLACE_OP((this_instr + 1), _NOP, 0, 0); + } + } + if (frame_pop(ctx, returning_code, returning_stacklevel)) { + break; + } stack_pointer = ctx->frame->stack_pointer; - /* Stack space handling */ - assert(corresponding_check_stack == NULL); - assert(co != NULL); - int framesize = co->co_framesize; - assert(framesize > 0); - assert(framesize <= curr_space); - curr_space -= framesize; - RELOAD_STACK(); res = temp; } op(_RETURN_GENERATOR, ( -- res)) { SYNC_SP(); - PyCodeObject *co = get_current_code_object(ctx); ctx->frame->stack_pointer = stack_pointer; - frame_pop(ctx); + PyCodeObject *returning_code = get_code_with_logging(this_instr); + if (returning_code == NULL) { + ctx->done = true; + break; + } + _Py_BloomFilter_Add(dependencies, returning_code); + int returning_stacklevel = this_instr->operand1; + if (frame_pop(ctx, returning_code, returning_stacklevel)) { + break; + } stack_pointer = ctx->frame->stack_pointer; res = sym_new_unknown(ctx); - - /* Stack space handling */ - assert(corresponding_check_stack == NULL); - assert(co != NULL); - int framesize = co->co_framesize; - assert(framesize > 0); - assert(framesize <= curr_space); - curr_space -= framesize; } - op(_YIELD_VALUE, (unused -- value)) { - value = sym_new_unknown(ctx); + op(_YIELD_VALUE, (retval -- value)) { + // Mimics PyStackRef_MakeHeapSafe in the interpreter. + JitOptRef temp = PyJitRef_StripReferenceInfo(retval); + DEAD(retval); + SAVE_STACK(); + ctx->frame->stack_pointer = stack_pointer; + PyCodeObject *returning_code = get_code_with_logging(this_instr); + if (returning_code == NULL) { + ctx->done = true; + break; + } + _Py_BloomFilter_Add(dependencies, returning_code); + int returning_stacklevel = this_instr->operand1; + if (frame_pop(ctx, returning_code, returning_stacklevel)) { + break; + } + stack_pointer = ctx->frame->stack_pointer; + RELOAD_STACK(); + value = temp; } op(_GET_ITER, (iterable -- iter, index_or_null)) { @@ -835,8 +875,6 @@ dummy_func(void) { } op(_CHECK_STACK_SPACE, (unused, unused, unused[oparg] -- unused, unused, unused[oparg])) { - assert(corresponding_check_stack == NULL); - corresponding_check_stack = this_instr; } op (_CHECK_STACK_SPACE_OPERAND, (framesize/2 -- )) { @@ -848,38 +886,29 @@ dummy_func(void) { op(_PUSH_FRAME, (new_frame -- )) { SYNC_SP(); - ctx->frame->stack_pointer = stack_pointer; + if (!CURRENT_FRAME_IS_INIT_SHIM()) { + ctx->frame->stack_pointer = stack_pointer; + } ctx->frame = (_Py_UOpsAbstractFrame *)PyJitRef_Unwrap(new_frame); ctx->curr_frame_depth++; stack_pointer = ctx->frame->stack_pointer; uint64_t operand = this_instr->operand0; - if (operand == 0 || (operand & 1)) { - // It's either a code object or NULL + if (operand == 0) { ctx->done = true; break; } - PyFunctionObject *func = (PyFunctionObject *)operand; - PyCodeObject *co = (PyCodeObject *)func->func_code; - assert(PyFunction_Check(func)); - ctx->frame->func = func; - /* Stack space handling */ - int framesize = co->co_framesize; - assert(framesize > 0); - curr_space += framesize; - if (curr_space < 0 || curr_space > INT32_MAX) { - // won't fit in signed 32-bit int - ctx->done = true; - break; - } - max_space = curr_space > max_space ? curr_space : max_space; - if (first_valid_check_stack == NULL) { - first_valid_check_stack = corresponding_check_stack; + if (!(operand & 1)) { + PyFunctionObject *func = (PyFunctionObject *)operand; + // No need to re-add to dependencies here. Already + // handled by the tracer. + ctx->frame->func = func; } - else if (corresponding_check_stack) { - // delete all but the first valid _CHECK_STACK_SPACE - corresponding_check_stack->opcode = _NOP; + // Fixed calls don't need IP guards. + if ((this_instr-1)->opcode == _SAVE_RETURN_OFFSET || + (this_instr-1)->opcode == _CREATE_INIT_FRAME) { + assert((this_instr+1)->opcode == _GUARD_IP__PUSH_FRAME); + REPLACE_OP(this_instr+1, _NOP, 0, 0); } - corresponding_check_stack = NULL; } op(_UNPACK_SEQUENCE, (seq -- values[oparg], top[0])) { @@ -1024,6 +1053,10 @@ dummy_func(void) { ctx->done = true; } + op(_DEOPT, (--)) { + ctx->done = true; + } + op(_REPLACE_WITH_TRUE, (value -- res)) { REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); res = sym_new_const(ctx, Py_True); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index b08099d8e2fc3b..01263fe8c7a78f 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -280,7 +280,6 @@ int already_bool = optimize_to_bool(this_instr, ctx, value, &value); if (!already_bool) { sym_set_type(value, &PyBool_Type); - value = sym_new_truthiness(ctx, value, true); } stack_pointer[-1] = value; break; @@ -1116,16 +1115,24 @@ JitOptRef temp = PyJitRef_StripReferenceInfo(retval); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); - PyCodeObject *co = get_current_code_object(ctx); ctx->frame->stack_pointer = stack_pointer; - frame_pop(ctx); + PyCodeObject *returning_code = get_code_with_logging(this_instr); + if (returning_code == NULL) { + ctx->done = true; + break; + } + int returning_stacklevel = this_instr->operand1; + if (ctx->curr_frame_depth >= 2) { + PyCodeObject *expected_code = ctx->frames[ctx->curr_frame_depth - 2].code; + if (expected_code == returning_code) { + assert((this_instr + 1)->opcode == _GUARD_IP_RETURN_VALUE); + REPLACE_OP((this_instr + 1), _NOP, 0, 0); + } + } + if (frame_pop(ctx, returning_code, returning_stacklevel)) { + break; + } stack_pointer = ctx->frame->stack_pointer; - assert(corresponding_check_stack == NULL); - assert(co != NULL); - int framesize = co->co_framesize; - assert(framesize > 0); - assert(framesize <= curr_space); - curr_space -= framesize; res = temp; stack_pointer[0] = res; stack_pointer += 1; @@ -1167,9 +1174,28 @@ } case _YIELD_VALUE: { + JitOptRef retval; JitOptRef value; - value = sym_new_unknown(ctx); - stack_pointer[-1] = value; + retval = stack_pointer[-1]; + JitOptRef temp = PyJitRef_StripReferenceInfo(retval); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + ctx->frame->stack_pointer = stack_pointer; + PyCodeObject *returning_code = get_code_with_logging(this_instr); + if (returning_code == NULL) { + ctx->done = true; + break; + } + _Py_BloomFilter_Add(dependencies, returning_code); + int returning_stacklevel = this_instr->operand1; + if (frame_pop(ctx, returning_code, returning_stacklevel)) { + break; + } + stack_pointer = ctx->frame->stack_pointer; + value = temp; + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -2103,6 +2129,8 @@ break; } + /* _JUMP_BACKWARD_NO_INTERRUPT is not a viable micro-op for tier 2 */ + case _GET_LEN: { JitOptRef obj; JitOptRef len; @@ -2557,8 +2585,6 @@ } case _CHECK_STACK_SPACE: { - assert(corresponding_check_stack == NULL); - corresponding_check_stack = this_instr; break; } @@ -2601,34 +2627,26 @@ new_frame = stack_pointer[-1]; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); - ctx->frame->stack_pointer = stack_pointer; + if (!CURRENT_FRAME_IS_INIT_SHIM()) { + ctx->frame->stack_pointer = stack_pointer; + } ctx->frame = (_Py_UOpsAbstractFrame *)PyJitRef_Unwrap(new_frame); ctx->curr_frame_depth++; stack_pointer = ctx->frame->stack_pointer; uint64_t operand = this_instr->operand0; - if (operand == 0 || (operand & 1)) { + if (operand == 0) { ctx->done = true; break; } - PyFunctionObject *func = (PyFunctionObject *)operand; - PyCodeObject *co = (PyCodeObject *)func->func_code; - assert(PyFunction_Check(func)); - ctx->frame->func = func; - int framesize = co->co_framesize; - assert(framesize > 0); - curr_space += framesize; - if (curr_space < 0 || curr_space > INT32_MAX) { - ctx->done = true; - break; - } - max_space = curr_space > max_space ? curr_space : max_space; - if (first_valid_check_stack == NULL) { - first_valid_check_stack = corresponding_check_stack; + if (!(operand & 1)) { + PyFunctionObject *func = (PyFunctionObject *)operand; + ctx->frame->func = func; } - else if (corresponding_check_stack) { - corresponding_check_stack->opcode = _NOP; + if ((this_instr-1)->opcode == _SAVE_RETURN_OFFSET || + (this_instr-1)->opcode == _CREATE_INIT_FRAME) { + assert((this_instr+1)->opcode == _GUARD_IP__PUSH_FRAME); + REPLACE_OP(this_instr+1, _NOP, 0, 0); } - corresponding_check_stack = NULL; break; } @@ -2761,9 +2779,24 @@ } case _CREATE_INIT_FRAME: { + JitOptRef *args; + JitOptRef self; JitOptRef init_frame; - init_frame = PyJitRef_NULL; - ctx->done = true; + args = &stack_pointer[-oparg]; + self = stack_pointer[-1 - oparg]; + ctx->frame->stack_pointer = stack_pointer - oparg - 2; + _Py_UOpsAbstractFrame *shim = frame_new(ctx, (PyCodeObject *)&_Py_InitCleanup, 0, NULL, 0); + if (shim == NULL) { + break; + } + shim->stack[0] = self; + shim->stack_pointer++; + assert((int)(shim->stack_pointer - shim->stack) == 1); + ctx->frame = shim; + ctx->curr_frame_depth++; + assert((this_instr + 1)->opcode == _PUSH_FRAME); + PyCodeObject *co = get_code_with_logging((this_instr + 1)); + init_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, args-1, oparg+1)); stack_pointer[-2 - oparg] = init_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -2948,8 +2981,13 @@ case _PY_FRAME_KW: { JitOptRef new_frame; - new_frame = PyJitRef_NULL; - ctx->done = true; + assert((this_instr + 2)->opcode == _PUSH_FRAME); + PyCodeObject *co = get_code_with_logging((this_instr + 2)); + if (co == NULL) { + ctx->done = true; + break; + } + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); stack_pointer[-3 - oparg] = new_frame; stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); @@ -3005,17 +3043,19 @@ case _RETURN_GENERATOR: { JitOptRef res; - PyCodeObject *co = get_current_code_object(ctx); ctx->frame->stack_pointer = stack_pointer; - frame_pop(ctx); + PyCodeObject *returning_code = get_code_with_logging(this_instr); + if (returning_code == NULL) { + ctx->done = true; + break; + } + _Py_BloomFilter_Add(dependencies, returning_code); + int returning_stacklevel = this_instr->operand1; + if (frame_pop(ctx, returning_code, returning_stacklevel)) { + break; + } stack_pointer = ctx->frame->stack_pointer; res = sym_new_unknown(ctx); - assert(corresponding_check_stack == NULL); - assert(co != NULL); - int framesize = co->co_framesize; - assert(framesize > 0); - assert(framesize <= curr_space); - curr_space -= framesize; stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -3265,6 +3305,10 @@ break; } + case _DYNAMIC_EXIT: { + break; + } + case _CHECK_VALIDITY: { break; } @@ -3399,6 +3443,7 @@ } case _DEOPT: { + ctx->done = true; break; } @@ -3418,3 +3463,23 @@ break; } + case _COLD_DYNAMIC_EXIT: { + break; + } + + case _GUARD_IP__PUSH_FRAME: { + break; + } + + case _GUARD_IP_YIELD_VALUE: { + break; + } + + case _GUARD_IP_RETURN_VALUE: { + break; + } + + case _GUARD_IP_RETURN_GENERATOR: { + break; + } + diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 01cff0b014cc7b..8a71eff465e5a3 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -817,9 +817,14 @@ _Py_uop_frame_new( JitOptRef *args, int arg_len) { - assert(ctx->curr_frame_depth < MAX_ABSTRACT_FRAME_DEPTH); + if (ctx->curr_frame_depth >= MAX_ABSTRACT_FRAME_DEPTH) { + ctx->done = true; + ctx->out_of_space = true; + OPT_STAT_INC(optimizer_frame_overflow); + return NULL; + } _Py_UOpsAbstractFrame *frame = &ctx->frames[ctx->curr_frame_depth]; - + frame->code = co; frame->stack_len = co->co_stacksize; frame->locals_len = co->co_nlocalsplus; @@ -901,13 +906,42 @@ _Py_uop_abstractcontext_init(JitOptContext *ctx) } int -_Py_uop_frame_pop(JitOptContext *ctx) +_Py_uop_frame_pop(JitOptContext *ctx, PyCodeObject *co, int curr_stackentries) { _Py_UOpsAbstractFrame *frame = ctx->frame; ctx->n_consumed = frame->locals; + ctx->curr_frame_depth--; - assert(ctx->curr_frame_depth >= 1); - ctx->frame = &ctx->frames[ctx->curr_frame_depth - 1]; + + if (ctx->curr_frame_depth >= 1) { + ctx->frame = &ctx->frames[ctx->curr_frame_depth - 1]; + + // We returned to the correct code. Nothing to do here. + if (co == ctx->frame->code) { + return 0; + } + // Else: the code we recorded doesn't match the code we *think* we're + // returning to. We could trace anything, we can't just return to the + // old frame. We have to restore what the tracer recorded + // as the traced next frame. + // Remove the current frame, and later swap it out with the right one. + else { + ctx->curr_frame_depth--; + } + } + // Else: trace stack underflow. + + // This handles swapping out frames. + assert(curr_stackentries >= 1); + // -1 to stackentries as we push to the stack our return value after this. + _Py_UOpsAbstractFrame *new_frame = _Py_uop_frame_new(ctx, co, curr_stackentries - 1, NULL, 0); + if (new_frame == NULL) { + ctx->done = true; + return 1; + } + + ctx->curr_frame_depth++; + ctx->frame = new_frame; return 0; } diff --git a/Python/pystate.c b/Python/pystate.c index 341c680a403608..c12a1418e74309 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -552,10 +552,6 @@ init_interpreter(PyInterpreterState *interp, _Py_brc_init_state(interp); #endif -#ifdef _Py_TIER2 - // Ensure the buffer is to be set as NULL. - interp->jit_uop_buffer = NULL; -#endif llist_init(&interp->mem_free_queue.head); llist_init(&interp->asyncio_tasks_head); interp->asyncio_tasks_lock = (PyMutex){0}; @@ -805,10 +801,6 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) #ifdef _Py_TIER2 _Py_ClearExecutorDeletionList(interp); - if (interp->jit_uop_buffer != NULL) { - _PyObject_VirtualFree(interp->jit_uop_buffer, UOP_BUFFER_SIZE); - interp->jit_uop_buffer = NULL; - } #endif _PyAST_Fini(interp); _PyAtExit_Fini(interp); @@ -831,6 +823,14 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) assert(cold->vm_data.warm); _PyExecutor_Free(cold); } + + struct _PyExecutorObject *cold_dynamic = interp->cold_dynamic_executor; + if (cold_dynamic != NULL) { + interp->cold_dynamic_executor = NULL; + assert(cold_dynamic->vm_data.valid); + assert(cold_dynamic->vm_data.warm); + _PyExecutor_Free(cold_dynamic); + } /* We don't clear sysdict and builtins until the end of this function. Because clearing other attributes can execute arbitrary Python code which requires sysdict and builtins. */ @@ -1501,6 +1501,9 @@ init_threadstate(_PyThreadStateImpl *_tstate, _tstate->asyncio_running_loop = NULL; _tstate->asyncio_running_task = NULL; +#ifdef _Py_TIER2 + _tstate->jit_tracer_state.code_buffer = NULL; +#endif tstate->delete_later = NULL; llist_init(&_tstate->mem_free_queue); @@ -1807,6 +1810,14 @@ tstate_delete_common(PyThreadState *tstate, int release_gil) assert(tstate_impl->refcounts.values == NULL); #endif +#if _Py_TIER2 + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + if (_tstate->jit_tracer_state.code_buffer != NULL) { + _PyObject_VirtualFree(_tstate->jit_tracer_state.code_buffer, UOP_BUFFER_SIZE); + _tstate->jit_tracer_state.code_buffer = NULL; + } +#endif + HEAD_UNLOCK(runtime); // XXX Unbind in PyThreadState_Clear(), or earlier diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 4621ad250f4633..bd4a8cf0d3e65c 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -359,6 +359,7 @@ Parser/parser.c - soft_keywords - Parser/lexer/lexer.c - type_comment_prefix - Python/ceval.c - _PyEval_BinaryOps - Python/ceval.c - _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS - +Python/ceval.c - _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR - Python/codecs.c - Py_hexdigits - Python/codecs.c - codecs_builtin_error_handlers - Python/codecs.c - ucnhash_capi - diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 9dd7e5dbfbae7b..d39013db4f7fd6 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -34,6 +34,8 @@ class Properties: side_exit: bool pure: bool uses_opcode: bool + needs_guard_ip: bool + unpredictable_jump: bool tier: int | None = None const_oparg: int = -1 needs_prev: bool = False @@ -75,6 +77,8 @@ def from_list(properties: list["Properties"]) -> "Properties": pure=all(p.pure for p in properties), needs_prev=any(p.needs_prev for p in properties), no_save_ip=all(p.no_save_ip for p in properties), + needs_guard_ip=any(p.needs_guard_ip for p in properties), + unpredictable_jump=any(p.unpredictable_jump for p in properties), ) @property @@ -102,6 +106,8 @@ def infallible(self) -> bool: side_exit=False, pure=True, no_save_ip=False, + needs_guard_ip=False, + unpredictable_jump=False, ) @@ -692,6 +698,11 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "PyStackRef_Wrap", "PyStackRef_Unwrap", "_PyLong_CheckExactAndCompact", + "_PyExecutor_FromExit", + "_PyJit_TryInitializeTracing", + "_Py_unset_eval_breaker_bit", + "_Py_set_eval_breaker_bit", + "trigger_backoff_counter", ) @@ -882,6 +893,46 @@ def stmt_escapes(stmt: Stmt) -> bool: else: assert False, "Unexpected statement type" +def stmt_has_jump_on_unpredictable_path_body(stmts: list[Stmt] | None, branches_seen: int) -> tuple[bool, int]: + if not stmts: + return False, branches_seen + predict = False + seen = 0 + for st in stmts: + predict_body, seen_body = stmt_has_jump_on_unpredictable_path(st, branches_seen) + predict = predict or predict_body + seen += seen_body + return predict, seen + +def stmt_has_jump_on_unpredictable_path(stmt: Stmt, branches_seen: int) -> tuple[bool, int]: + if isinstance(stmt, BlockStmt): + return stmt_has_jump_on_unpredictable_path_body(stmt.body, branches_seen) + elif isinstance(stmt, SimpleStmt): + for tkn in stmt.contents: + if tkn.text == "JUMPBY": + return True, branches_seen + return False, branches_seen + elif isinstance(stmt, IfStmt): + predict, seen = stmt_has_jump_on_unpredictable_path(stmt.body, branches_seen) + if stmt.else_body: + predict_else, seen_else = stmt_has_jump_on_unpredictable_path(stmt.else_body, branches_seen) + return predict != predict_else, seen + seen_else + 1 + return predict, seen + 1 + elif isinstance(stmt, MacroIfStmt): + predict, seen = stmt_has_jump_on_unpredictable_path_body(stmt.body, branches_seen) + if stmt.else_body: + predict_else, seen_else = stmt_has_jump_on_unpredictable_path_body(stmt.else_body, branches_seen) + return predict != predict_else, seen + seen_else + return predict, seen + elif isinstance(stmt, ForStmt): + unpredictable, branches_seen = stmt_has_jump_on_unpredictable_path(stmt.body, branches_seen) + return unpredictable, branches_seen + 1 + elif isinstance(stmt, WhileStmt): + unpredictable, branches_seen = stmt_has_jump_on_unpredictable_path(stmt.body, branches_seen) + return unpredictable, branches_seen + 1 + else: + assert False, f"Unexpected statement type {stmt}" + def compute_properties(op: parser.CodeDef) -> Properties: escaping_calls = find_escaping_api_calls(op) @@ -909,6 +960,8 @@ def compute_properties(op: parser.CodeDef) -> Properties: escapes = stmt_escapes(op.block) pure = False if isinstance(op, parser.LabelDef) else "pure" in op.annotations no_save_ip = False if isinstance(op, parser.LabelDef) else "no_save_ip" in op.annotations + unpredictable, branches_seen = stmt_has_jump_on_unpredictable_path(op.block, 0) + unpredictable_jump = False if isinstance(op, parser.LabelDef) else (unpredictable and branches_seen > 0) return Properties( escaping_calls=escaping_calls, escapes=escapes, @@ -932,6 +985,11 @@ def compute_properties(op: parser.CodeDef) -> Properties: no_save_ip=no_save_ip, tier=tier_variable(op), needs_prev=variable_used(op, "prev_instr"), + needs_guard_ip=(isinstance(op, parser.InstDef) + and (unpredictable_jump and "replaced" not in op.annotations)) + or variable_used(op, "LOAD_IP") + or variable_used(op, "DISPATCH_INLINED"), + unpredictable_jump=unpredictable_jump, ) def expand(items: list[StackItem], oparg: int) -> list[StackItem]: diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 61e855eb003706..0b5f764ec52b45 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -7,6 +7,7 @@ analysis_error, Label, CodeSection, + Uop, ) from cwriter import CWriter from typing import Callable, TextIO, Iterator, Iterable @@ -107,8 +108,9 @@ class Emitter: labels: dict[str, Label] _replacers: dict[str, ReplacementFunctionType] cannot_escape: bool + jump_prefix: str - def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = False): + def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = False, jump_prefix: str = ""): self._replacers = { "EXIT_IF": self.exit_if, "AT_END_EXIT_IF": self.exit_if_after, @@ -131,6 +133,7 @@ def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = self.out = out self.labels = labels self.cannot_escape = cannot_escape + self.jump_prefix = jump_prefix def dispatch( self, @@ -167,7 +170,7 @@ def deopt_if( family_name = inst.family.name self.emit(f"UPDATE_MISS_STATS({family_name});\n") self.emit(f"assert(_PyOpcode_Deopt[opcode] == ({family_name}));\n") - self.emit(f"JUMP_TO_PREDICTED({family_name});\n") + self.emit(f"JUMP_TO_PREDICTED({self.jump_prefix}{family_name});\n") self.emit("}\n") return not always_true(first_tkn) @@ -198,10 +201,10 @@ def exit_if_after( def goto_error(self, offset: int, storage: Storage) -> str: if offset > 0: - return f"JUMP_TO_LABEL(pop_{offset}_error);" + return f"{self.jump_prefix}JUMP_TO_LABEL(pop_{offset}_error);" if offset < 0: storage.copy().flush(self.out) - return f"JUMP_TO_LABEL(error);" + return f"{self.jump_prefix}JUMP_TO_LABEL(error);" def error_if( self, @@ -421,7 +424,7 @@ def goto_label(self, goto: Token, label: Token, storage: Storage) -> None: elif storage.spilled: raise analysis_error("Cannot jump from spilled label without reloading the stack pointer", goto) self.out.start_line() - self.out.emit("JUMP_TO_LABEL(") + self.out.emit(f"{self.jump_prefix}JUMP_TO_LABEL(") self.out.emit(label) self.out.emit(")") @@ -731,6 +734,10 @@ def cflags(p: Properties) -> str: flags.append("HAS_PURE_FLAG") if p.no_save_ip: flags.append("HAS_NO_SAVE_IP_FLAG") + if p.unpredictable_jump: + flags.append("HAS_UNPREDICTABLE_JUMP_FLAG") + if p.needs_guard_ip: + flags.append("HAS_NEEDS_GUARD_IP_FLAG") if flags: return " | ".join(flags) else: diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index b649b38123388d..21ae785a0ec445 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -56,6 +56,8 @@ "ERROR_NO_POP", "NO_SAVE_IP", "PERIODIC", + "UNPREDICTABLE_JUMP", + "NEEDS_GUARD_IP", ] @@ -201,7 +203,7 @@ def generate_metadata_table(analysis: Analysis, out: CWriter) -> None: out.emit("struct opcode_metadata {\n") out.emit("uint8_t valid_entry;\n") out.emit("uint8_t instr_format;\n") - out.emit("uint16_t flags;\n") + out.emit("uint32_t flags;\n") out.emit("};\n\n") out.emit( f"extern const struct opcode_metadata _PyOpcode_opcode_metadata[{table_size}];\n" diff --git a/Tools/cases_generator/target_generator.py b/Tools/cases_generator/target_generator.py index 324ef2773abe28..36fa1d7fa4908b 100644 --- a/Tools/cases_generator/target_generator.py +++ b/Tools/cases_generator/target_generator.py @@ -31,6 +31,16 @@ def write_opcode_targets(analysis: Analysis, out: CWriter) -> None: for target in targets: out.emit(target) out.emit("};\n") + targets = ["&&_unknown_opcode,\n"] * 256 + for name, op in analysis.opmap.items(): + if op < 256: + targets[op] = f"&&record_previous_inst,\n" + out.emit("#if _Py_TIER2\n") + out.emit("static void *opcode_tracing_targets_table[256] = {\n") + for target in targets: + out.emit(target) + out.emit("};\n") + out.emit(f"#endif\n") out.emit("#else /* _Py_TAIL_CALL_INTERP */\n") def function_proto(name: str) -> str: @@ -38,7 +48,9 @@ def function_proto(name: str) -> str: def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: - out.emit("static py_tail_call_funcptr instruction_funcptr_table[256];\n") + out.emit("static py_tail_call_funcptr instruction_funcptr_handler_table[256];\n") + out.emit("\n") + out.emit("static py_tail_call_funcptr instruction_funcptr_tracing_table[256];\n") out.emit("\n") # Emit function prototypes for labels. @@ -60,7 +72,7 @@ def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: out.emit("\n") # Emit the dispatch table. - out.emit("static py_tail_call_funcptr instruction_funcptr_table[256] = {\n") + out.emit("static py_tail_call_funcptr instruction_funcptr_handler_table[256] = {\n") for name in sorted(analysis.instructions.keys()): out.emit(f"[{name}] = _TAIL_CALL_{name},\n") named_values = analysis.opmap.values() @@ -68,6 +80,16 @@ def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: if rest not in named_values: out.emit(f"[{rest}] = _TAIL_CALL_UNKNOWN_OPCODE,\n") out.emit("};\n") + + # Emit the tracing dispatch table. + out.emit("static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = {\n") + for name in sorted(analysis.instructions.keys()): + out.emit(f"[{name}] = _TAIL_CALL_record_previous_inst,\n") + named_values = analysis.opmap.values() + for rest in range(256): + if rest not in named_values: + out.emit(f"[{rest}] = _TAIL_CALL_UNKNOWN_OPCODE,\n") + out.emit("};\n") outfile.write("#endif /* _Py_TAIL_CALL_INTERP */\n") arg_parser = argparse.ArgumentParser( diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index 1bb5f48658ddfc..ac3e6b94afe49e 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -63,6 +63,7 @@ class Tier2Emitter(Emitter): def __init__(self, out: CWriter, labels: dict[str, Label]): super().__init__(out, labels) self._replacers["oparg"] = self.oparg + self._replacers["IP_OFFSET_OF"] = self.ip_offset_of def goto_error(self, offset: int, storage: Storage) -> str: # To do: Add jump targets for popping values. @@ -134,10 +135,30 @@ def oparg( self.out.emit_at(uop.name[-1], tkn) return True + def ip_offset_of( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + assert uop.name.startswith("_GUARD_IP") + # LPAREN + next(tkn_iter) + tok = next(tkn_iter) + self.emit(f" OFFSET_OF_{tok.text};\n") + # RPAREN + next(tkn_iter) + # SEMI + next(tkn_iter) + return True -def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> Stack: +def write_uop(uop: Uop, emitter: Emitter, stack: Stack, offset_strs: dict[str, tuple[str, str]]) -> Stack: locals: dict[str, Local] = {} try: + if name_offset_pair := offset_strs.get(uop.name): + emitter.emit(f"#define OFFSET_OF_{name_offset_pair[0]} ({name_offset_pair[1]})\n") emitter.out.start_line() if uop.properties.oparg: emitter.emit("oparg = CURRENT_OPARG();\n") @@ -158,6 +179,8 @@ def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> Stack: idx += 1 _, storage = emitter.emit_tokens(uop, storage, None, False) storage.flush(emitter.out) + if name_offset_pair: + emitter.emit(f"#undef OFFSET_OF_{name_offset_pair[0]}\n") except StackError as ex: raise analysis_error(ex.args[0], uop.body.open) from None return storage.stack @@ -165,6 +188,29 @@ def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> Stack: SKIPS = ("_EXTENDED_ARG",) +def populate_offset_strs(analysis: Analysis) -> dict[str, tuple[str, str]]: + offset_strs: dict[str, tuple[str, str]] = {} + for name, uop in analysis.uops.items(): + if not f"_GUARD_IP_{name}" in analysis.uops: + continue + tkn_iter = uop.body.tokens() + found = False + offset_str = "" + for token in tkn_iter: + if token.kind == "IDENTIFIER" and token.text == "LOAD_IP": + if found: + raise analysis_error("Cannot have two LOAD_IP in a guarded single uop.", uop.body.open) + offset = [] + while token.kind != "SEMI": + offset.append(token.text) + token = next(tkn_iter) + # 1: to remove the LOAD_IP text + offset_str = "".join(offset[1:]) + found = True + assert offset_str + offset_strs[f"_GUARD_IP_{name}"] = (name, offset_str) + return offset_strs + def generate_tier2( filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool ) -> None: @@ -179,7 +225,9 @@ def generate_tier2( ) out = CWriter(outfile, 2, lines) emitter = Tier2Emitter(out, analysis.labels) + offset_strs = populate_offset_strs(analysis) out.emit("\n") + for name, uop in analysis.uops.items(): if uop.properties.tier == 1: continue @@ -194,13 +242,15 @@ def generate_tier2( out.emit(f"case {uop.name}: {{\n") declare_variables(uop, out) stack = Stack() - stack = write_uop(uop, emitter, stack) + stack = write_uop(uop, emitter, stack, offset_strs) out.start_line() if not uop.properties.always_exits: out.emit("break;\n") out.start_line() out.emit("}") out.emit("\n\n") + + out.emit("\n") outfile.write("#undef TIER_TWO\n") diff --git a/Tools/cases_generator/uop_metadata_generator.py b/Tools/cases_generator/uop_metadata_generator.py index 1cc23837a72dea..0e0396e5143348 100644 --- a/Tools/cases_generator/uop_metadata_generator.py +++ b/Tools/cases_generator/uop_metadata_generator.py @@ -23,13 +23,13 @@ def generate_names_and_flags(analysis: Analysis, out: CWriter) -> None: - out.emit("extern const uint16_t _PyUop_Flags[MAX_UOP_ID+1];\n") + out.emit("extern const uint32_t _PyUop_Flags[MAX_UOP_ID+1];\n") out.emit("typedef struct _rep_range { uint8_t start; uint8_t stop; } ReplicationRange;\n") out.emit("extern const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1];\n") out.emit("extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1];\n\n") out.emit("extern int _PyUop_num_popped(int opcode, int oparg);\n\n") out.emit("#ifdef NEED_OPCODE_METADATA\n") - out.emit("const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {\n") + out.emit("const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = {\n") for uop in analysis.uops.values(): if uop.is_viable() and uop.properties.tier != 1: out.emit(f"[{uop.name}] = {cflags(uop.properties)},\n") diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 2f146014a1c26b..857e926d119900 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -55,13 +55,10 @@ do { \ __attribute__((musttail)) return jitted(frame, stack_pointer, tstate); \ } while (0) -#undef GOTO_TIER_ONE -#define GOTO_TIER_ONE(TARGET) \ -do { \ - tstate->current_executor = NULL; \ - _PyFrame_SetStackPointer(frame, stack_pointer); \ - return TARGET; \ -} while (0) +#undef GOTO_TIER_ONE_SETUP +#define GOTO_TIER_ONE_SETUP \ + tstate->current_executor = NULL; \ + _PyFrame_SetStackPointer(frame, stack_pointer); #undef LOAD_IP #define LOAD_IP(UNUSED) \ From 209eaff68c3b241c01aece14182cb9ced51526fc Mon Sep 17 00:00:00 2001 From: dr-carlos <77367421+dr-carlos@users.noreply.github.com> Date: Fri, 14 Nov 2025 04:47:17 +1030 Subject: [PATCH 037/638] gh-137969: Fix double evaluation of `ForwardRef`s which rely on globals (#140974) --- Lib/annotationlib.py | 39 +++++++++------- Lib/test/test_annotationlib.py | 45 +++++++++++++++++++ ...-11-04-15-40-35.gh-issue-137969.9VZQVt.rst | 3 ++ 3 files changed, 72 insertions(+), 15 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-04-15-40-35.gh-issue-137969.9VZQVt.rst diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 2166dbff0ee70c..33907b1fc2a53a 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -150,33 +150,42 @@ def evaluate( if globals is None: globals = {} + if type_params is None and owner is not None: + type_params = getattr(owner, "__type_params__", None) + if locals is None: locals = {} if isinstance(owner, type): locals.update(vars(owner)) + elif ( + type_params is not None + or isinstance(self.__cell__, dict) + or self.__extra_names__ + ): + # Create a new locals dict if necessary, + # to avoid mutating the argument. + locals = dict(locals) - if type_params is None and owner is not None: - # "Inject" type parameters into the local namespace - # (unless they are shadowed by assignments *in* the local namespace), - # as a way of emulating annotation scopes when calling `eval()` - type_params = getattr(owner, "__type_params__", None) - - # Type parameters exist in their own scope, which is logically - # between the locals and the globals. We simulate this by adding - # them to the globals. Similar reasoning applies to nonlocals stored in cells. - if type_params is not None or isinstance(self.__cell__, dict): - globals = dict(globals) + # "Inject" type parameters into the local namespace + # (unless they are shadowed by assignments *in* the local namespace), + # as a way of emulating annotation scopes when calling `eval()` if type_params is not None: for param in type_params: - globals[param.__name__] = param + locals.setdefault(param.__name__, param) + + # Similar logic can be used for nonlocals, which should not + # override locals. if isinstance(self.__cell__, dict): - for cell_name, cell_value in self.__cell__.items(): + for cell_name, cell in self.__cell__.items(): try: - globals[cell_name] = cell_value.cell_contents + cell_value = cell.cell_contents except ValueError: pass + else: + locals.setdefault(cell_name, cell_value) + if self.__extra_names__: - locals = {**locals, **self.__extra_names__} + locals.update(self.__extra_names__) arg = self.__forward_arg__ if arg.isidentifier() and not keyword.iskeyword(arg): diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 9f3275d5071484..8208d0e9c94819 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -2149,6 +2149,51 @@ def test_fwdref_invalid_syntax(self): with self.assertRaises(SyntaxError): fr.evaluate() + def test_re_evaluate_generics(self): + global global_alias + + # If we've already run this test before, + # ensure the variable is still undefined + if "global_alias" in globals(): + del global_alias + + class C: + x: global_alias[int] + + # Evaluate the ForwardRef once + evaluated = get_annotations(C, format=Format.FORWARDREF)["x"].evaluate( + format=Format.FORWARDREF + ) + + # Now define the global and ensure that the ForwardRef evaluates + global_alias = list + self.assertEqual(evaluated.evaluate(), list[int]) + + def test_fwdref_evaluate_argument_mutation(self): + class C[T]: + nonlocal alias + x: alias[T] + + # Mutable arguments + globals_ = globals() + globals_copy = globals_.copy() + locals_ = locals() + locals_copy = locals_.copy() + + # Evaluate the ForwardRef, ensuring we use __cell__ and type params + get_annotations(C, format=Format.FORWARDREF)["x"].evaluate( + globals=globals_, + locals=locals_, + type_params=C.__type_params__, + format=Format.FORWARDREF, + ) + + # Check if the passed in mutable arguments equal the originals + self.assertEqual(globals_, globals_copy) + self.assertEqual(locals_, locals_copy) + + alias = list + def test_fwdref_final_class(self): with self.assertRaises(TypeError): class C(ForwardRef): diff --git a/Misc/NEWS.d/next/Library/2025-11-04-15-40-35.gh-issue-137969.9VZQVt.rst b/Misc/NEWS.d/next/Library/2025-11-04-15-40-35.gh-issue-137969.9VZQVt.rst new file mode 100644 index 00000000000000..dfa582bdbc8825 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-04-15-40-35.gh-issue-137969.9VZQVt.rst @@ -0,0 +1,3 @@ +Fix :meth:`annotationlib.ForwardRef.evaluate` returning +:class:`~annotationlib.ForwardRef` objects which don't update with new +globals. From a486d452c78a7dfcd42561f6c151bf1fef0a756e Mon Sep 17 00:00:00 2001 From: Osama Abdelkader <78818069+osamakader@users.noreply.github.com> Date: Thu, 13 Nov 2025 20:05:28 +0100 Subject: [PATCH 038/638] gh-140601: Add ResourceWarning to iterparse when not closed (GH-140603) When iterparse() opens a file by filename and is not explicitly closed, emit a ResourceWarning to alert developers of the resource leak. Signed-off-by: Osama Abdelkader Co-authored-by: Serhiy Storchaka --- Doc/library/xml.etree.elementtree.rst | 4 ++ Doc/whatsnew/3.15.rst | 6 +++ Lib/test/test_xml_etree.py | 47 +++++++++++++++++++ Lib/xml/etree/ElementTree.py | 12 +++-- ...-10-25-22-55-07.gh-issue-140601.In3MlS.rst | 4 ++ 5 files changed, 69 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-10-25-22-55-07.gh-issue-140601.In3MlS.rst diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index 881708a4dd702e..cbbc87b4721a9f 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -656,6 +656,10 @@ Functions .. versionchanged:: 3.13 Added the :meth:`!close` method. + .. versionchanged:: next + A :exc:`ResourceWarning` is now emitted if the iterator opened a file + and is not explicitly closed. + .. function:: parse(source, parser=None) diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 895616e3049a50..31594a2e70bd4c 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -1244,3 +1244,9 @@ that may require changes to your code. * :meth:`~mmap.mmap.resize` has been removed on platforms that don't support the underlying syscall, instead of raising a :exc:`SystemError`. + +* Resource warning is now emitted for unclosed + :func:`xml.etree.ElementTree.iterparse` iterator if it opened a file. + Use its :meth:`!close` method or the :func:`contextlib.closing` context + manager to close it. + (Contributed by Osama Abdelkader and Serhiy Storchaka in :gh:`140601`.) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 25c084c8b9c9eb..87811199706a1f 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -1436,18 +1436,40 @@ def test_nonexistent_file(self): def test_resource_warnings_not_exhausted(self): # Not exhausting the iterator still closes the underlying file (bpo-43292) + # Not closing before del should emit ResourceWarning it = ET.iterparse(SIMPLE_XMLFILE) with warnings_helper.check_no_resource_warning(self): + it.close() + del it + gc_collect() + + it = ET.iterparse(SIMPLE_XMLFILE) + with self.assertWarns(ResourceWarning) as wm: del it gc_collect() + # Not 'unclosed file'. + self.assertIn('unclosed iterparse iterator', str(wm.warning)) + self.assertIn(repr(SIMPLE_XMLFILE), str(wm.warning)) + self.assertEqual(wm.filename, __file__) it = ET.iterparse(SIMPLE_XMLFILE) with warnings_helper.check_no_resource_warning(self): action, elem = next(it) + it.close() self.assertEqual((action, elem.tag), ('end', 'element')) del it, elem gc_collect() + it = ET.iterparse(SIMPLE_XMLFILE) + with self.assertWarns(ResourceWarning) as wm: + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + del it, elem + gc_collect() + self.assertIn('unclosed iterparse iterator', str(wm.warning)) + self.assertIn(repr(SIMPLE_XMLFILE), str(wm.warning)) + self.assertEqual(wm.filename, __file__) + def test_resource_warnings_failed_iteration(self): self.addCleanup(os_helper.unlink, TESTFN) with open(TESTFN, "wb") as f: @@ -1461,15 +1483,40 @@ def test_resource_warnings_failed_iteration(self): next(it) self.assertEqual(str(cm.exception), 'junk after document element: line 1, column 12') + it.close() del cm, it gc_collect() + it = ET.iterparse(TESTFN) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'document')) + with self.assertWarns(ResourceWarning) as wm: + with self.assertRaises(ET.ParseError) as cm: + next(it) + self.assertEqual(str(cm.exception), + 'junk after document element: line 1, column 12') + del cm, it + gc_collect() + self.assertIn('unclosed iterparse iterator', str(wm.warning)) + self.assertIn(repr(TESTFN), str(wm.warning)) + self.assertEqual(wm.filename, __file__) + def test_resource_warnings_exhausted(self): it = ET.iterparse(SIMPLE_XMLFILE) with warnings_helper.check_no_resource_warning(self): + list(it) + it.close() + del it + gc_collect() + + it = ET.iterparse(SIMPLE_XMLFILE) + with self.assertWarns(ResourceWarning) as wm: list(it) del it gc_collect() + self.assertIn('unclosed iterparse iterator', str(wm.warning)) + self.assertIn(repr(SIMPLE_XMLFILE), str(wm.warning)) + self.assertEqual(wm.filename, __file__) def test_close_not_exhausted(self): iterparse = ET.iterparse diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index dafe5b1b8a0c3f..d8c0b1b621684b 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -1261,16 +1261,20 @@ def iterator(source): gen = iterator(source) class IterParseIterator(collections.abc.Iterator): __next__ = gen.__next__ + def close(self): + nonlocal close_source if close_source: source.close() + close_source = False gen.close() - def __del__(self): - # TODO: Emit a ResourceWarning if it was not explicitly closed. - # (When the close() method will be supported in all maintained Python versions.) + def __del__(self, _warn=warnings.warn): if close_source: - source.close() + try: + _warn(f"unclosed iterparse iterator {source.name!r}", ResourceWarning, stacklevel=2) + finally: + source.close() it = IterParseIterator() it.root = None diff --git a/Misc/NEWS.d/next/Library/2025-10-25-22-55-07.gh-issue-140601.In3MlS.rst b/Misc/NEWS.d/next/Library/2025-10-25-22-55-07.gh-issue-140601.In3MlS.rst new file mode 100644 index 00000000000000..72666bb8224d63 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-25-22-55-07.gh-issue-140601.In3MlS.rst @@ -0,0 +1,4 @@ +:func:`xml.etree.ElementTree.iterparse` now emits a :exc:`ResourceWarning` +when the iterator is not explicitly closed and was opened with a filename. +This helps developers identify and fix resource leaks. Patch by Osama +Abdelkader. From 4885ecfbda4cc792691e5d488ef6cb09727eb417 Mon Sep 17 00:00:00 2001 From: M Bussonnier Date: Fri, 14 Nov 2025 04:18:54 +0100 Subject: [PATCH 039/638] gh-140790: pdb: Initialize instance variables in Pdb.__init__ (#140791) Initialize lineno, stack, curindex, curframe, currentbp, and _user_requested_quit attributes in `Pdb.__init__``. --- Lib/pdb.py | 10 ++++++++-- .../2025-10-30-12-36-19.gh-issue-140790._3T6-N.rst | 1 + 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-10-30-12-36-19.gh-issue-140790._3T6-N.rst diff --git a/Lib/pdb.py b/Lib/pdb.py index fdc74198582eec..b799a113503502 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -398,6 +398,12 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, self._current_task = None + self.lineno = None + self.stack = [] + self.curindex = 0 + self.curframe = None + self._user_requested_quit = False + def set_trace(self, frame=None, *, commands=None): Pdb._last_pdb_instance = self if frame is None: @@ -474,7 +480,7 @@ def forget(self): self.lineno = None self.stack = [] self.curindex = 0 - if hasattr(self, 'curframe') and self.curframe: + if self.curframe: self.curframe.f_globals.pop('__pdb_convenience_variables', None) self.curframe = None self.tb_lineno.clear() @@ -1493,7 +1499,7 @@ def checkline(self, filename, lineno, module_globals=None): """ # this method should be callable before starting debugging, so default # to "no globals" if there is no current frame - frame = getattr(self, 'curframe', None) + frame = self.curframe if module_globals is None: module_globals = frame.f_globals if frame else None line = linecache.getline(filename, lineno, module_globals) diff --git a/Misc/NEWS.d/next/Library/2025-10-30-12-36-19.gh-issue-140790._3T6-N.rst b/Misc/NEWS.d/next/Library/2025-10-30-12-36-19.gh-issue-140790._3T6-N.rst new file mode 100644 index 00000000000000..03856f0b9b6d0a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-30-12-36-19.gh-issue-140790._3T6-N.rst @@ -0,0 +1 @@ +Initialize all Pdb's instance variables in ``__init__``, remove some hasattr/getattr From a4dd66275b62453bec055d730a8ce7173e519b6d Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 14 Nov 2025 10:38:49 +0100 Subject: [PATCH 040/638] gh-140550: Use a bool for the Py_mod_gil value (GH-141519) This needs a single bit, but was stored as a void* in the module struct. This didn't matter due to packing, but now that there's another bool in the struct, we can save a bit of memory by making md_gil a bool. Variables that changed type are renamed, to detect conflicts. --- Include/internal/pycore_moduleobject.h | 2 +- Lib/test/test_sys.py | 2 +- Objects/moduleobject.c | 15 ++++++++------- Python/import.c | 26 ++++++++++++++------------ 4 files changed, 24 insertions(+), 21 deletions(-) diff --git a/Include/internal/pycore_moduleobject.h b/Include/internal/pycore_moduleobject.h index 6eef6eaa5df844..9a62daf6621ca2 100644 --- a/Include/internal/pycore_moduleobject.h +++ b/Include/internal/pycore_moduleobject.h @@ -30,7 +30,7 @@ typedef struct { PyObject *md_name; bool md_token_is_def; /* if true, `md_token` is the PyModuleDef */ #ifdef Py_GIL_DISABLED - void *md_gil; + bool md_requires_gil; #endif Py_ssize_t md_state_size; traverseproc md_state_traverse; diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 798f58737b1bf6..2f169c1165df05 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1725,7 +1725,7 @@ def get_gen(): yield 1 check(int(PyLong_BASE**2), vsize('') + 3*self.longdigit) # module if support.Py_GIL_DISABLED: - md_gil = 'P' + md_gil = '?' else: md_gil = '' check(unittest, size('PPPP?' + md_gil + 'NPPPPP')) diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 9dee03bdb5ee55..6c1c5f5eb89c0c 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -178,7 +178,7 @@ new_module_notrack(PyTypeObject *mt) m->md_name = NULL; m->md_token_is_def = false; #ifdef Py_GIL_DISABLED - m->md_gil = Py_MOD_GIL_USED; + m->md_requires_gil = true; #endif m->md_state_size = 0; m->md_state_traverse = NULL; @@ -361,7 +361,7 @@ _PyModule_CreateInitialized(PyModuleDef* module, int module_api_version) m->md_token_is_def = true; module_copy_members_from_deflike(m, module); #ifdef Py_GIL_DISABLED - m->md_gil = Py_MOD_GIL_USED; + m->md_requires_gil = true; #endif return (PyObject*)m; } @@ -380,7 +380,7 @@ module_from_def_and_spec( int has_multiple_interpreters_slot = 0; void *multiple_interpreters = (void *)0; int has_gil_slot = 0; - void *gil_slot = Py_MOD_GIL_USED; + bool requires_gil = true; int has_execution_slots = 0; const char *name; int ret; @@ -474,7 +474,7 @@ module_from_def_and_spec( name); goto error; } - gil_slot = cur_slot->value; + requires_gil = (cur_slot->value != Py_MOD_GIL_NOT_USED); has_gil_slot = 1; break; case Py_mod_abi: @@ -581,9 +581,9 @@ module_from_def_and_spec( mod->md_token = token; } #ifdef Py_GIL_DISABLED - mod->md_gil = gil_slot; + mod->md_requires_gil = requires_gil; #else - (void)gil_slot; + (void)requires_gil; #endif mod->md_exec = m_exec; } else { @@ -664,11 +664,12 @@ PyModule_FromSlotsAndSpec(const PyModuleDef_Slot *slots, PyObject *spec) int PyUnstable_Module_SetGIL(PyObject *module, void *gil) { + bool requires_gil = (gil != Py_MOD_GIL_NOT_USED); if (!PyModule_Check(module)) { PyErr_BadInternalCall(); return -1; } - ((PyModuleObject *)module)->md_gil = gil; + ((PyModuleObject *)module)->md_requires_gil = requires_gil; return 0; } #endif diff --git a/Python/import.c b/Python/import.c index 2afa7c15e6a8dc..b05b40448d02ac 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1017,9 +1017,10 @@ struct extensions_cache_value { _Py_ext_module_origin origin; #ifdef Py_GIL_DISABLED - /* The module's md_gil slot, for legacy modules that are reinitialized from - m_dict rather than calling their initialization function again. */ - void *md_gil; + /* The module's md_requires_gil member, for legacy modules that are + * reinitialized from m_dict rather than calling their initialization + * function again. */ + bool md_requires_gil; #endif }; @@ -1350,7 +1351,7 @@ static struct extensions_cache_value * _extensions_cache_set(PyObject *path, PyObject *name, PyModuleDef *def, PyModInitFunction m_init, Py_ssize_t m_index, PyObject *m_dict, - _Py_ext_module_origin origin, void *md_gil) + _Py_ext_module_origin origin, bool requires_gil) { struct extensions_cache_value *value = NULL; void *key = NULL; @@ -1405,11 +1406,11 @@ _extensions_cache_set(PyObject *path, PyObject *name, /* m_dict is set by set_cached_m_dict(). */ .origin=origin, #ifdef Py_GIL_DISABLED - .md_gil=md_gil, + .md_requires_gil=requires_gil, #endif }; #ifndef Py_GIL_DISABLED - (void)md_gil; + (void)requires_gil; #endif if (init_cached_m_dict(newvalue, m_dict) < 0) { goto finally; @@ -1547,7 +1548,8 @@ _PyImport_CheckGILForModule(PyObject* module, PyObject *module_name) } if (!PyModule_Check(module) || - ((PyModuleObject *)module)->md_gil == Py_MOD_GIL_USED) { + ((PyModuleObject *)module)->md_requires_gil) + { if (_PyEval_EnableGILPermanent(tstate)) { int warn_result = PyErr_WarnFormat( PyExc_RuntimeWarning, @@ -1725,7 +1727,7 @@ struct singlephase_global_update { Py_ssize_t m_index; PyObject *m_dict; _Py_ext_module_origin origin; - void *md_gil; + bool md_requires_gil; }; static struct extensions_cache_value * @@ -1784,7 +1786,7 @@ update_global_state_for_extension(PyThreadState *tstate, #endif cached = _extensions_cache_set( path, name, def, m_init, singlephase->m_index, m_dict, - singlephase->origin, singlephase->md_gil); + singlephase->origin, singlephase->md_requires_gil); if (cached == NULL) { // XXX Ignore this error? Doing so would effectively // mark the module as not loadable. @@ -1873,7 +1875,7 @@ reload_singlephase_extension(PyThreadState *tstate, if (def->m_base.m_copy != NULL) { // For non-core modules, fetch the GIL slot that was stored by // import_run_extension(). - ((PyModuleObject *)mod)->md_gil = cached->md_gil; + ((PyModuleObject *)mod)->md_requires_gil = cached->md_requires_gil; } #endif /* We can't set mod->md_def if it's missing, @@ -2128,7 +2130,7 @@ import_run_extension(PyThreadState *tstate, PyModInitFunction p0, .m_index=def->m_base.m_index, .origin=info->origin, #ifdef Py_GIL_DISABLED - .md_gil=((PyModuleObject *)mod)->md_gil, + .md_requires_gil=((PyModuleObject *)mod)->md_requires_gil, #endif }; // gh-88216: Extensions and def->m_base.m_copy can be updated @@ -2323,7 +2325,7 @@ _PyImport_FixupBuiltin(PyThreadState *tstate, PyObject *mod, const char *name, .origin=_Py_ext_module_origin_CORE, #ifdef Py_GIL_DISABLED /* Unused when m_dict == NULL. */ - .md_gil=NULL, + .md_requires_gil=false, #endif }; cached = update_global_state_for_extension( From 1e4e59bb3714ba7c6b6297f1a74e231b056f004c Mon Sep 17 00:00:00 2001 From: Itamar Oren Date: Fri, 14 Nov 2025 01:43:25 -0800 Subject: [PATCH 041/638] gh-116146: Add C-API to create module from spec and initfunc (GH-139196) Co-authored-by: Kumar Aditya Co-authored-by: Petr Viktorin Co-authored-by: Victor Stinner --- Doc/c-api/import.rst | 21 ++++ Doc/whatsnew/3.15.rst | 4 + Include/cpython/import.h | 7 ++ Lib/test/test_embed.py | 25 ++++ ...-11-08-10-51-50.gh-issue-116146.pCmx6L.rst | 2 + Programs/_testembed.c | 111 ++++++++++++++++++ Python/import.c | 74 ++++++++---- 7 files changed, 223 insertions(+), 21 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 8eabc0406b11ce..24e673d3d1394f 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -333,3 +333,24 @@ Importing Modules strings instead of Python :class:`str` objects. .. versionadded:: 3.14 + +.. c:function:: PyObject* PyImport_CreateModuleFromInitfunc(PyObject *spec, PyObject* (*initfunc)(void)) + + This function is a building block that enables embedders to implement + the :py:meth:`~importlib.abc.Loader.create_module` step of custom + static extension importers (e.g. of statically-linked extensions). + + *spec* must be a :class:`~importlib.machinery.ModuleSpec` object. + + *initfunc* must be an :ref:`initialization function `, + the same as for :c:func:`PyImport_AppendInittab`. + + On success, create and return a module object. + This module will not be initialized; call :c:func:`!PyModule_Exec` + to initialize it. + (Custom importers should do this in their + :py:meth:`~importlib.abc.Loader.exec_module` method.) + + On error, return NULL with an exception set. + + .. versionadded:: next diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 31594a2e70bd4c..9393b65ed8e906 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -1080,6 +1080,10 @@ New features thread state. (Contributed by Victor Stinner in :gh:`139653`.) +* Add a new :c:func:`PyImport_CreateModuleFromInitfunc` C-API for creating + a module from a *spec* and *initfunc*. + (Contributed by Itamar Oren in :gh:`116146`.) + Changed C APIs -------------- diff --git a/Include/cpython/import.h b/Include/cpython/import.h index 0ce0b1ee6cce2a..149a20af8b9cbb 100644 --- a/Include/cpython/import.h +++ b/Include/cpython/import.h @@ -10,6 +10,13 @@ struct _inittab { PyAPI_DATA(struct _inittab *) PyImport_Inittab; PyAPI_FUNC(int) PyImport_ExtendInittab(struct _inittab *newtab); +// Custom importers may use this API to initialize statically linked +// extension modules directly from a spec and init function, +// without needing to go through inittab +PyAPI_FUNC(PyObject *) PyImport_CreateModuleFromInitfunc( + PyObject *spec, + PyObject *(*initfunc)(void)); + struct _frozen { const char *name; /* ASCII encoded string */ const unsigned char *code; diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 1933f691a78be5..1078796eae84e2 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -239,6 +239,31 @@ def test_repeated_init_and_inittab(self): lines = "\n".join(lines) + "\n" self.assertEqual(out, lines) + def test_create_module_from_initfunc(self): + out, err = self.run_embedded_interpreter("test_create_module_from_initfunc") + if support.Py_GIL_DISABLED: + # the test imports a singlephase init extension, so it emits a warning + # under the free-threaded build + expected_runtime_warning = ( + "RuntimeWarning: The global interpreter lock (GIL)" + " has been enabled to load module 'embedded_ext'" + ) + filtered_err_lines = [ + line + for line in err.strip().splitlines() + if expected_runtime_warning not in line + ] + self.assertEqual(filtered_err_lines, []) + else: + self.assertEqual(err, "") + self.assertEqual(out, + "\n" + "my_test_extension.executed='yes'\n" + "my_test_extension.exec_slot_ran='yes'\n" + "\n" + "embedded_ext.executed='yes'\n" + ) + def test_forced_io_encoding(self): # Checks forced configuration of embedded interpreter IO streams env = dict(os.environ, PYTHONIOENCODING="utf-8:surrogateescape") diff --git a/Misc/NEWS.d/next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst b/Misc/NEWS.d/next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst new file mode 100644 index 00000000000000..be8043e26ddda8 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst @@ -0,0 +1,2 @@ +Add a new :c:func:`PyImport_CreateModuleFromInitfunc` C-API for creating a +module from a *spec* and *initfunc*. Patch by Itamar Oren. diff --git a/Programs/_testembed.c b/Programs/_testembed.c index d3600fecbe2775..27224e508bdd3e 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -166,6 +166,8 @@ static PyModuleDef embedded_ext = { static PyObject* PyInit_embedded_ext(void) { + // keep this as a single-phase initialization module; + // see test_create_module_from_initfunc return PyModule_Create(&embedded_ext); } @@ -1894,8 +1896,16 @@ static int test_initconfig_exit(void) } +int +extension_module_exec(PyObject *mod) +{ + return PyModule_AddStringConstant(mod, "exec_slot_ran", "yes"); +} + + static PyModuleDef_Slot extension_slots[] = { {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {Py_mod_exec, extension_module_exec}, {0, NULL} }; @@ -2213,6 +2223,106 @@ static int test_repeated_init_and_inittab(void) return 0; } +static PyObject* +create_module(PyObject* self, PyObject* spec) +{ + PyObject *name = PyObject_GetAttrString(spec, "name"); + if (!name) { + return NULL; + } + if (PyUnicode_EqualToUTF8(name, "my_test_extension")) { + Py_DECREF(name); + return PyImport_CreateModuleFromInitfunc(spec, init_my_test_extension); + } + if (PyUnicode_EqualToUTF8(name, "embedded_ext")) { + Py_DECREF(name); + return PyImport_CreateModuleFromInitfunc(spec, PyInit_embedded_ext); + } + PyErr_Format(PyExc_LookupError, "static module %R not found", name); + Py_DECREF(name); + return NULL; +} + +static PyObject* +exec_module(PyObject* self, PyObject* mod) +{ + if (PyModule_Exec(mod) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyMethodDef create_static_module_methods[] = { + {"create_module", create_module, METH_O, NULL}, + {"exec_module", exec_module, METH_O, NULL}, + {} +}; + +static struct PyModuleDef create_static_module_def = { + PyModuleDef_HEAD_INIT, + .m_name = "create_static_module", + .m_size = 0, + .m_methods = create_static_module_methods, + .m_slots = extension_slots, +}; + +PyMODINIT_FUNC PyInit_create_static_module(void) { + return PyModuleDef_Init(&create_static_module_def); +} + +static int +test_create_module_from_initfunc(void) +{ + wchar_t* argv[] = { + PROGRAM_NAME, + L"-c", + // Multi-phase initialization + L"import my_test_extension;" + L"print(my_test_extension);" + L"print(f'{my_test_extension.executed=}');" + L"print(f'{my_test_extension.exec_slot_ran=}');" + // Single-phase initialization + L"import embedded_ext;" + L"print(embedded_ext);" + L"print(f'{embedded_ext.executed=}');" + }; + PyConfig config; + if (PyImport_AppendInittab("create_static_module", + &PyInit_create_static_module) != 0) { + fprintf(stderr, "PyImport_AppendInittab() failed\n"); + return 1; + } + PyConfig_InitPythonConfig(&config); + config.isolated = 1; + config_set_argv(&config, Py_ARRAY_LENGTH(argv), argv); + init_from_config_clear(&config); + int result = PyRun_SimpleString( + "import sys\n" + "from importlib.util import spec_from_loader\n" + "import create_static_module\n" + "class StaticExtensionImporter:\n" + " _ORIGIN = \"static-extension\"\n" + " @classmethod\n" + " def find_spec(cls, fullname, path, target=None):\n" + " if fullname in {'my_test_extension', 'embedded_ext'}:\n" + " return spec_from_loader(fullname, cls, origin=cls._ORIGIN)\n" + " return None\n" + " @staticmethod\n" + " def create_module(spec):\n" + " return create_static_module.create_module(spec)\n" + " @staticmethod\n" + " def exec_module(module):\n" + " create_static_module.exec_module(module)\n" + " module.executed = 'yes'\n" + "sys.meta_path.append(StaticExtensionImporter)\n" + ); + if (result < 0) { + fprintf(stderr, "PyRun_SimpleString() failed\n"); + return 1; + } + return Py_RunMain(); +} + static void wrap_allocator(PyMemAllocatorEx *allocator); static void unwrap_allocator(PyMemAllocatorEx *allocator); @@ -2396,6 +2506,7 @@ static struct TestCase TestCases[] = { #endif {"test_get_incomplete_frame", test_get_incomplete_frame}, {"test_gilstate_after_finalization", test_gilstate_after_finalization}, + {"test_create_module_from_initfunc", test_create_module_from_initfunc}, {NULL, NULL} }; diff --git a/Python/import.c b/Python/import.c index b05b40448d02ac..9ab2d3b3552235 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2364,8 +2364,23 @@ is_builtin(PyObject *name) return 0; } +static PyModInitFunction +lookup_inittab_initfunc(const struct _Py_ext_module_loader_info* info) +{ + for (struct _inittab *p = INITTAB; p->name != NULL; p++) { + if (_PyUnicode_EqualToASCIIString(info->name, p->name)) { + return (PyModInitFunction)p->initfunc; + } + } + // not found + return NULL; +} + static PyObject* -create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) +create_builtin( + PyThreadState *tstate, PyObject *name, + PyObject *spec, + PyModInitFunction initfunc) { struct _Py_ext_module_loader_info info; if (_Py_ext_module_loader_info_init_for_builtin(&info, name) < 0) { @@ -2396,25 +2411,15 @@ create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) _extensions_cache_delete(info.path, info.name); } - struct _inittab *found = NULL; - for (struct _inittab *p = INITTAB; p->name != NULL; p++) { - if (_PyUnicode_EqualToASCIIString(info.name, p->name)) { - found = p; - break; - } - } - if (found == NULL) { - // not found - mod = Py_NewRef(Py_None); - goto finally; - } - - PyModInitFunction p0 = (PyModInitFunction)found->initfunc; + PyModInitFunction p0 = initfunc; if (p0 == NULL) { - /* Cannot re-init internal module ("sys" or "builtins") */ - assert(is_core_module(tstate->interp, info.name, info.path)); - mod = import_add_module(tstate, info.name); - goto finally; + p0 = lookup_inittab_initfunc(&info); + if (p0 == NULL) { + /* Cannot re-init internal module ("sys" or "builtins") */ + assert(is_core_module(tstate->interp, info.name, info.path)); + mod = import_add_module(tstate, info.name); + goto finally; + } } #ifdef Py_GIL_DISABLED @@ -2440,6 +2445,33 @@ create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) return mod; } +PyObject* +PyImport_CreateModuleFromInitfunc( + PyObject *spec, PyObject *(*initfunc)(void)) +{ + if (initfunc == NULL) { + PyErr_BadInternalCall(); + return NULL; + } + + PyThreadState *tstate = _PyThreadState_GET(); + + PyObject *name = PyObject_GetAttr(spec, &_Py_ID(name)); + if (name == NULL) { + return NULL; + } + + if (!PyUnicode_Check(name)) { + PyErr_Format(PyExc_TypeError, + "spec name must be string, not %T", name); + Py_DECREF(name); + return NULL; + } + + PyObject *mod = create_builtin(tstate, name, spec, initfunc); + Py_DECREF(name); + return mod; +} /*****************************/ /* the builtin modules table */ @@ -3209,7 +3241,7 @@ bootstrap_imp(PyThreadState *tstate) } // Create the _imp module from its definition. - PyObject *mod = create_builtin(tstate, name, spec); + PyObject *mod = create_builtin(tstate, name, spec, NULL); Py_CLEAR(name); Py_DECREF(spec); if (mod == NULL) { @@ -4369,7 +4401,7 @@ _imp_create_builtin(PyObject *module, PyObject *spec) return NULL; } - PyObject *mod = create_builtin(tstate, name, spec); + PyObject *mod = create_builtin(tstate, name, spec, NULL); Py_DECREF(name); return mod; } From 181a2f4f2e3bed8dc6be5630e9bfb3362194ab3a Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Fri, 14 Nov 2025 11:59:19 +0200 Subject: [PATCH 042/638] gh-139596: Cease caching config.cache & ccache in GH Actions (#141451) --- .github/workflows/build.yml | 5 ----- .github/workflows/reusable-context.yml | 9 --------- .github/workflows/reusable-macos.yml | 3 --- .github/workflows/reusable-san.yml | 3 --- .github/workflows/reusable-ubuntu.yml | 3 --- .github/workflows/reusable-wasi.yml | 6 +----- .gitignore | 1 - 7 files changed, 1 insertion(+), 29 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a0f60c30ac8a60..8e15400e4978eb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -205,7 +205,6 @@ jobs: free-threading: true uses: ./.github/workflows/reusable-macos.yml with: - config_hash: ${{ needs.build-context.outputs.config-hash }} free-threading: ${{ matrix.free-threading }} os: ${{ matrix.os }} @@ -237,7 +236,6 @@ jobs: bolt: true uses: ./.github/workflows/reusable-ubuntu.yml with: - config_hash: ${{ needs.build-context.outputs.config-hash }} bolt-optimizations: ${{ matrix.bolt }} free-threading: ${{ matrix.free-threading }} os: ${{ matrix.os }} @@ -414,8 +412,6 @@ jobs: needs: build-context if: needs.build-context.outputs.run-tests == 'true' uses: ./.github/workflows/reusable-wasi.yml - with: - config_hash: ${{ needs.build-context.outputs.config-hash }} test-hypothesis: name: "Hypothesis tests on Ubuntu" @@ -600,7 +596,6 @@ jobs: uses: ./.github/workflows/reusable-san.yml with: sanitizer: ${{ matrix.sanitizer }} - config_hash: ${{ needs.build-context.outputs.config-hash }} free-threading: ${{ matrix.free-threading }} cross-build-linux: diff --git a/.github/workflows/reusable-context.yml b/.github/workflows/reusable-context.yml index d2668ddcac1a3d..66c7cc47de03fb 100644 --- a/.github/workflows/reusable-context.yml +++ b/.github/workflows/reusable-context.yml @@ -17,9 +17,6 @@ on: # yamllint disable-line rule:truthy # || 'falsy-branch' # }} # - config-hash: - description: Config hash value for use in cache keys - value: ${{ jobs.compute-changes.outputs.config-hash }} # str run-docs: description: Whether to build the docs value: ${{ jobs.compute-changes.outputs.run-docs }} # bool @@ -42,7 +39,6 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 outputs: - config-hash: ${{ steps.config-hash.outputs.hash }} run-ci-fuzz: ${{ steps.changes.outputs.run-ci-fuzz }} run-docs: ${{ steps.changes.outputs.run-docs }} run-tests: ${{ steps.changes.outputs.run-tests }} @@ -100,8 +96,3 @@ jobs: GITHUB_EVENT_NAME: ${{ github.event_name }} CCF_TARGET_REF: ${{ github.base_ref || github.event.repository.default_branch }} CCF_HEAD_REF: ${{ github.event.pull_request.head.sha || github.sha }} - - - name: Compute hash for config cache key - id: config-hash - run: | - echo "hash=${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }}" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index d85c46b96f873d..98d557ba1eab84 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -3,9 +3,6 @@ name: Reusable macOS on: workflow_call: inputs: - config_hash: - required: true - type: string free-threading: required: false type: boolean diff --git a/.github/workflows/reusable-san.yml b/.github/workflows/reusable-san.yml index 7fe96d1b238b04..c601d0b73380d4 100644 --- a/.github/workflows/reusable-san.yml +++ b/.github/workflows/reusable-san.yml @@ -6,9 +6,6 @@ on: sanitizer: required: true type: string - config_hash: - required: true - type: string free-threading: description: Whether to use free-threaded mode required: false diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 7b93b5f51b00df..0c1ebe29ae322f 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -3,9 +3,6 @@ name: Reusable Ubuntu on: workflow_call: inputs: - config_hash: - required: true - type: string bolt-optimizations: description: Whether to enable BOLT optimizations required: false diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index 8f412288f530bc..a309ef4e7f4485 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -2,10 +2,6 @@ name: Reusable WASI on: workflow_call: - inputs: - config_hash: - required: true - type: string env: FORCE_COLOR: 1 @@ -53,7 +49,7 @@ jobs: - name: "Configure build Python" run: python3 Tools/wasm/wasi configure-build-python -- --config-cache --with-pydebug - name: "Make build Python" - run: python3 Tools/wasm/wasi.py make-build-python + run: python3 Tools/wasm/wasi make-build-python - name: "Configure host" # `--with-pydebug` inferred from configure-build-python run: python3 Tools/wasm/wasi configure-host -- --config-cache diff --git a/.gitignore b/.gitignore index 2bf4925647ddcd..4ea2fd9655471d 100644 --- a/.gitignore +++ b/.gitignore @@ -135,7 +135,6 @@ Tools/unicode/data/ /config.log /config.status /config.status.lineno -# hendrikmuhs/ccache-action@v1 /.ccache /cross-build/ /jit_stencils*.h From 3bacae55980561cb99095a20a70c45d6174e056d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 14 Nov 2025 11:13:24 +0100 Subject: [PATCH 043/638] gh-131510: Use PyUnstable_Unicode_GET_CACHED_HASH() (GH-141520) Replace code that directly accesses PyASCIIObject.hash with PyUnstable_Unicode_GET_CACHED_HASH(). Remove redundant "assert(PyUnicode_Check(op))" from PyUnstable_Unicode_GET_CACHED_HASH(), _PyASCIIObject_CAST() already implements the check. --- Include/cpython/unicodeobject.h | 1 - Include/internal/pycore_object.h | 3 +-- Objects/dictobject.c | 3 +-- Objects/typeobject.c | 2 +- 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index 73e3bc44d6c9ca..2853d24c34b66e 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -301,7 +301,6 @@ static inline Py_ssize_t PyUnicode_GET_LENGTH(PyObject *op) { /* Returns the cached hash, or -1 if not cached yet. */ static inline Py_hash_t PyUnstable_Unicode_GET_CACHED_HASH(PyObject *op) { - assert(PyUnicode_Check(op)); #ifdef Py_GIL_DISABLED return _Py_atomic_load_ssize_relaxed(&_PyASCIIObject_CAST(op)->hash); #else diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 980d6d7764bd2c..fb50acd62da5eb 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -863,8 +863,7 @@ static inline Py_hash_t _PyObject_HashFast(PyObject *op) { if (PyUnicode_CheckExact(op)) { - Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED( - _PyASCIIObject_CAST(op)->hash); + Py_hash_t hash = PyUnstable_Unicode_GET_CACHED_HASH(op); if (hash != -1) { return hash; } diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 65eed151c2829d..14de21f3c67210 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -400,8 +400,7 @@ static int _PyObject_InlineValuesConsistencyCheck(PyObject *obj); static inline Py_hash_t unicode_get_hash(PyObject *o) { - assert(PyUnicode_CheckExact(o)); - return FT_ATOMIC_LOAD_SSIZE_RELAXED(_PyASCIIObject_CAST(o)->hash); + return PyUnstable_Unicode_GET_CACHED_HASH(o); } /* Print summary info about the state of the optimized allocator */ diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 58228d6248522e..61bcc21ce13d47 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -6036,7 +6036,7 @@ static PyObject * update_cache(struct type_cache_entry *entry, PyObject *name, unsigned int version_tag, PyObject *value) { _Py_atomic_store_ptr_relaxed(&entry->value, value); /* borrowed */ - assert(_PyASCIIObject_CAST(name)->hash != -1); + assert(PyUnstable_Unicode_GET_CACHED_HASH(name) != -1); OBJECT_STAT_INC_COND(type_cache_collisions, entry->name != Py_None && entry->name != name); // We're releasing this under the lock for simplicity sake because it's always a // exact unicode object or Py_None so it's safe to do so. From 5ac0b55ebc792936184f8e08697e60d5b3f8b946 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 14 Nov 2025 11:22:18 +0100 Subject: [PATCH 044/638] gh-141376: Remove exceptions from `make smelly` (GH-141392) * Don't ignore initialized data and BSS * Remove exceptions for _init and _fini --- Tools/build/smelly.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/Tools/build/smelly.py b/Tools/build/smelly.py index 9a360412a73a4d..424fa6ad4a1371 100755 --- a/Tools/build/smelly.py +++ b/Tools/build/smelly.py @@ -21,8 +21,6 @@ }) IGNORED_EXTENSION = "_ctypes_test" -# Ignore constructor and destructor functions -IGNORED_SYMBOLS = {'_init', '_fini'} def is_local_symbol_type(symtype): @@ -34,19 +32,12 @@ def is_local_symbol_type(symtype): if symtype.islower() and symtype not in "uvw": return True - # Ignore the initialized data section (d and D) and the BSS data - # section. For example, ignore "__bss_start (type: B)" - # and "_edata (type: D)". - if symtype in "bBdD": - return True - return False def get_exported_symbols(library, dynamic=False): print(f"Check that {library} only exports symbols starting with Py or _Py") - # Only look at dynamic symbols args = ['nm', '--no-sort'] if dynamic: args.append('--dynamic') @@ -89,8 +80,6 @@ def get_smelly_symbols(stdout, dynamic=False): if is_local_symbol_type(symtype): local_symbols.append(result) - elif symbol in IGNORED_SYMBOLS: - local_symbols.append(result) else: smelly_symbols.append(result) From ef90261be508b97d682589aac8f00065a9585683 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Fri, 14 Nov 2025 11:20:36 +0000 Subject: [PATCH 045/638] gh-141004: Document `PyOS_InterruptOccurred` (GH-141526) --- Doc/c-api/sys.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index 336e3ef96400f4..ee73c1c8adaa7b 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -123,6 +123,24 @@ Operating System Utilities This is a thin wrapper around either :c:func:`!sigaction` or :c:func:`!signal`. Do not call those functions directly! + +.. c:function:: int PyOS_InterruptOccurred(void) + + Check if a :c:macro:`!SIGINT` signal has been received. + + Returns ``1`` if a :c:macro:`!SIGINT` has occurred and clears the signal flag, + or ``0`` otherwise. + + In most cases, you should prefer :c:func:`PyErr_CheckSignals` over this function. + :c:func:`!PyErr_CheckSignals` invokes the appropriate signal handlers + for all pending signals, allowing Python code to handle the signal properly. + This function only detects :c:macro:`!SIGINT` and does not invoke any Python + signal handlers. + + This function is async-signal-safe and this function cannot fail. + The caller must hold an :term:`attached thread state`. + + .. c:function:: wchar_t* Py_DecodeLocale(const char* arg, size_t *size) .. warning:: From c10fa5be6167b1338ad194f9fe4be4782e025175 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Fri, 14 Nov 2025 09:22:36 -0500 Subject: [PATCH 046/638] gh-131229: Temporarily skip `test_basic_multiple_interpreters_deleted_no_reset` (GH-141552) This is a temporary band-aid to unblock other PRs. Co-authored-by: Kumar Aditya --- Lib/test/test_import/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index fe669bb04df02a..fd9750eae80445 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -3261,6 +3261,7 @@ def test_basic_multiple_interpreters_main_no_reset(self): # * m_copy was copied from interp2 (was from interp1) # * module's global state was updated, not reset + @unittest.skip("gh-131229: This is suddenly very flaky") @no_rerun(reason="rerun not possible; module state is never cleared (see gh-102251)") @requires_subinterpreters def test_basic_multiple_interpreters_deleted_no_reset(self): From 8deaa9393eadf84e6e571be611e0c5a377abf7cd Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 14 Nov 2025 16:49:28 +0200 Subject: [PATCH 047/638] gh-122255: Synchronize warnings in C and Python implementations of the warnings module (GH-122824) In the linecache module and in the Python implementation of the warnings module, a DeprecationWarning is issued when m.__loader__ differs from m.__spec__.loader (like in the C implementation of the warnings module). --- Lib/linecache.py | 65 +++++++++++++++---- Lib/test/test_linecache.py | 32 +++++++-- Lib/test/test_warnings/__init__.py | 5 +- ...-08-08-12-39-36.gh-issue-122255.J_gU8Y.rst | 4 ++ 4 files changed, 83 insertions(+), 23 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-08-08-12-39-36.gh-issue-122255.J_gU8Y.rst diff --git a/Lib/linecache.py b/Lib/linecache.py index ef3b2d9136b4d2..b5bf9dbdd3cbc7 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -224,21 +224,58 @@ def lazycache(filename, module_globals): def _make_lazycache_entry(filename, module_globals): if not filename or (filename.startswith('<') and filename.endswith('>')): return None - # Try for a __loader__, if available - if module_globals and '__name__' in module_globals: - spec = module_globals.get('__spec__') - name = getattr(spec, 'name', None) or module_globals['__name__'] - loader = getattr(spec, 'loader', None) - if loader is None: - loader = module_globals.get('__loader__') - get_source = getattr(loader, 'get_source', None) - - if name and get_source: - def get_lines(name=name, *args, **kwargs): - return get_source(name, *args, **kwargs) - return (get_lines,) - return None + if module_globals is not None and not isinstance(module_globals, dict): + raise TypeError(f'module_globals must be a dict, not {type(module_globals).__qualname__}') + if not module_globals or '__name__' not in module_globals: + return None + + spec = module_globals.get('__spec__') + name = getattr(spec, 'name', None) or module_globals['__name__'] + if name is None: + return None + + loader = _bless_my_loader(module_globals) + if loader is None: + return None + + get_source = getattr(loader, 'get_source', None) + if get_source is None: + return None + + def get_lines(name=name, *args, **kwargs): + return get_source(name, *args, **kwargs) + return (get_lines,) + +def _bless_my_loader(module_globals): + # Similar to _bless_my_loader() in importlib._bootstrap_external, + # but always emits warnings instead of errors. + loader = module_globals.get('__loader__') + if loader is None and '__spec__' not in module_globals: + return None + spec = module_globals.get('__spec__') + + # The __main__ module has __spec__ = None. + if spec is None and module_globals.get('__name__') == '__main__': + return loader + + spec_loader = getattr(spec, 'loader', None) + if spec_loader is None: + import warnings + warnings.warn( + 'Module globals is missing a __spec__.loader', + DeprecationWarning) + return loader + + assert spec_loader is not None + if loader is not None and loader != spec_loader: + import warnings + warnings.warn( + 'Module globals; __loader__ != __spec__.loader', + DeprecationWarning) + return loader + + return spec_loader def _register_code(code, string, name): diff --git a/Lib/test/test_linecache.py b/Lib/test/test_linecache.py index 02f65338428c8f..fcd94edc611fac 100644 --- a/Lib/test/test_linecache.py +++ b/Lib/test/test_linecache.py @@ -259,22 +259,44 @@ def raise_memoryerror(*args, **kwargs): def test_loader(self): filename = 'scheme://path' - for loader in (None, object(), NoSourceLoader()): + linecache.clearcache() + module_globals = {'__name__': 'a.b.c', '__loader__': None} + self.assertEqual(linecache.getlines(filename, module_globals), []) + + for loader in object(), NoSourceLoader(): linecache.clearcache() module_globals = {'__name__': 'a.b.c', '__loader__': loader} - self.assertEqual(linecache.getlines(filename, module_globals), []) + with self.assertWarns(DeprecationWarning) as w: + self.assertEqual(linecache.getlines(filename, module_globals), []) + self.assertEqual(str(w.warning), + 'Module globals is missing a __spec__.loader') linecache.clearcache() module_globals = {'__name__': 'a.b.c', '__loader__': FakeLoader()} - self.assertEqual(linecache.getlines(filename, module_globals), - ['source for a.b.c\n']) + with self.assertWarns(DeprecationWarning) as w: + self.assertEqual(linecache.getlines(filename, module_globals), + ['source for a.b.c\n']) + self.assertEqual(str(w.warning), + 'Module globals is missing a __spec__.loader') - for spec in (None, object(), ModuleSpec('', FakeLoader())): + for spec in None, object(): linecache.clearcache() module_globals = {'__name__': 'a.b.c', '__loader__': FakeLoader(), '__spec__': spec} + with self.assertWarns(DeprecationWarning) as w: + self.assertEqual(linecache.getlines(filename, module_globals), + ['source for a.b.c\n']) + self.assertEqual(str(w.warning), + 'Module globals is missing a __spec__.loader') + + linecache.clearcache() + module_globals = {'__name__': 'a.b.c', '__loader__': FakeLoader(), + '__spec__': ModuleSpec('', FakeLoader())} + with self.assertWarns(DeprecationWarning) as w: self.assertEqual(linecache.getlines(filename, module_globals), ['source for a.b.c\n']) + self.assertEqual(str(w.warning), + 'Module globals; __loader__ != __spec__.loader') linecache.clearcache() spec = ModuleSpec('x.y.z', FakeLoader()) diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index e6666ddc638037..a6af5057cc8968 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -727,7 +727,7 @@ def check_module_globals(self, module_globals): def check_module_globals_error(self, module_globals, errmsg, errtype=ValueError): if self.module is py_warnings: - self.check_module_globals(module_globals) + self.check_module_globals_deprecated(module_globals, errmsg) return with self.module.catch_warnings(record=True) as w: self.module.filterwarnings('always') @@ -738,9 +738,6 @@ def check_module_globals_error(self, module_globals, errmsg, errtype=ValueError) self.assertEqual(len(w), 0) def check_module_globals_deprecated(self, module_globals, msg): - if self.module is py_warnings: - self.check_module_globals(module_globals) - return with self.module.catch_warnings(record=True) as w: self.module.filterwarnings('always') self.module.warn_explicit( diff --git a/Misc/NEWS.d/next/Library/2024-08-08-12-39-36.gh-issue-122255.J_gU8Y.rst b/Misc/NEWS.d/next/Library/2024-08-08-12-39-36.gh-issue-122255.J_gU8Y.rst new file mode 100644 index 00000000000000..63e71c19f8b084 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-08-08-12-39-36.gh-issue-122255.J_gU8Y.rst @@ -0,0 +1,4 @@ +In the :mod:`linecache` module and in the Python implementation of the +:mod:`warnings` module, a ``DeprecationWarning`` is issued when +``mod.__loader__`` differs from ``mod.__spec__.loader`` (like in the C +implementation of the :mod:`!warnings` module). From 49e74210cb652d8bd538a4cc887f507396cfc893 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 14 Nov 2025 15:50:03 +0100 Subject: [PATCH 048/638] gh-139344: Remove pending removal notice for undeprecated importlib.resources API (GH-141507) --- Doc/deprecations/pending-removal-in-3.13.rst | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/Doc/deprecations/pending-removal-in-3.13.rst b/Doc/deprecations/pending-removal-in-3.13.rst index 2fd2f12cc6a2c4..d5b8c80e8f9aa0 100644 --- a/Doc/deprecations/pending-removal-in-3.13.rst +++ b/Doc/deprecations/pending-removal-in-3.13.rst @@ -38,15 +38,3 @@ APIs: * :meth:`!unittest.TestProgram.usageExit` (:gh:`67048`) * :class:`!webbrowser.MacOSX` (:gh:`86421`) * :class:`classmethod` descriptor chaining (:gh:`89519`) -* :mod:`importlib.resources` deprecated methods: - - * ``contents()`` - * ``is_resource()`` - * ``open_binary()`` - * ``open_text()`` - * ``path()`` - * ``read_binary()`` - * ``read_text()`` - - Use :func:`importlib.resources.files` instead. Refer to `importlib-resources: Migrating from Legacy - `_ (:gh:`106531`) From 10bec7c1eb3ee27f490a067426eef452b15f78f9 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Fri, 14 Nov 2025 19:52:01 +0500 Subject: [PATCH 049/638] GH-141312: Allow only integers to longrangeiter_setstate state (GH-141317) This fixes an assertion error when the new computed start is not an integer. --- Lib/test/test_range.py | 10 ++++++++++ .../2025-11-10-23-07-06.gh-issue-141312.H-58GB.rst | 2 ++ Objects/rangeobject.c | 5 +++++ 3 files changed, 17 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-10-23-07-06.gh-issue-141312.H-58GB.rst diff --git a/Lib/test/test_range.py b/Lib/test/test_range.py index 3870b153688b25..2c9c290e8906b7 100644 --- a/Lib/test/test_range.py +++ b/Lib/test/test_range.py @@ -470,6 +470,16 @@ def test_iterator_setstate(self): it.__setstate__(2**64 - 7) self.assertEqual(list(it), [12, 10]) + def test_iterator_invalid_setstate(self): + for invalid_value in (1.0, ""): + ranges = (('rangeiter', range(10, 100, 2)), + ('longrangeiter', range(10, 2**65, 2))) + for rng_name, rng in ranges: + with self.subTest(invalid_value=invalid_value, range=rng_name): + it = iter(rng) + with self.assertRaises(TypeError): + it.__setstate__(invalid_value) + def test_odd_bug(self): # This used to raise a "SystemError: NULL result without error" # because the range validation step was eating the exception diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-10-23-07-06.gh-issue-141312.H-58GB.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-10-23-07-06.gh-issue-141312.H-58GB.rst new file mode 100644 index 00000000000000..fdb136cef3f33c --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-10-23-07-06.gh-issue-141312.H-58GB.rst @@ -0,0 +1,2 @@ +Fix the assertion failure in the ``__setstate__`` method of the range iterator +when a non-integer argument is passed. Patch by Sergey Miryanov. diff --git a/Objects/rangeobject.c b/Objects/rangeobject.c index f8cdfe68a6435e..e93346fb27703f 100644 --- a/Objects/rangeobject.c +++ b/Objects/rangeobject.c @@ -1042,6 +1042,11 @@ longrangeiter_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) static PyObject * longrangeiter_setstate(PyObject *op, PyObject *state) { + if (!PyLong_CheckExact(state)) { + PyErr_Format(PyExc_TypeError, "state must be an int, not %T", state); + return NULL; + } + longrangeiterobject *r = (longrangeiterobject*)op; PyObject *zero = _PyLong_GetZero(); // borrowed reference int cmp; From fa245df4a0848c15cf8d907c10fc92819994b866 Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Fri, 14 Nov 2025 19:55:04 +0500 Subject: [PATCH 050/638] GH-141509: Fix warning about remaining subinterpreters (GH-141528) Co-authored-by: Peter Bierma --- Lib/test/test_interpreters/test_api.py | 2 +- .../2025-11-14-00-19-45.gh-issue-141528.VWdax1.rst | 3 +++ Python/pylifecycle.c | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-00-19-45.gh-issue-141528.VWdax1.rst diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index 9a5ee03e4722c0..fd9e46bf335fad 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -432,7 +432,7 @@ def test_cleanup_in_repl(self): exit()""" stdout, stderr = repl.communicate(script) self.assertIsNone(stderr) - self.assertIn(b"remaining subinterpreters", stdout) + self.assertIn(b"Interpreter.close()", stdout) self.assertNotIn(b"Traceback", stdout) @support.requires_subprocess() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-00-19-45.gh-issue-141528.VWdax1.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-00-19-45.gh-issue-141528.VWdax1.rst new file mode 100644 index 00000000000000..a51aa49522866b --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-00-19-45.gh-issue-141528.VWdax1.rst @@ -0,0 +1,3 @@ +Suggest using :meth:`concurrent.interpreters.Interpreter.close` instead of the +private ``_interpreters.destroy`` function when warning about remaining subinterpreters. +Patch by Sergey Miryanov. diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 805805ef188e83..67368b5ce077aa 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -2643,7 +2643,7 @@ finalize_subinterpreters(void) (void)PyErr_WarnEx( PyExc_RuntimeWarning, "remaining subinterpreters; " - "destroy them with _interpreters.destroy()", + "close them with Interpreter.close()", 0); /* Swap out the current tstate, which we know must belong From a415a1812c4d7798131d077c8776503bb3e1844f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 14 Nov 2025 15:56:37 +0100 Subject: [PATCH 051/638] gh-139653: Remove assertions in _Py_InitializeRecursionLimits() (#141551) These checks were invalid and failed randomly on FreeBSD and Alpine Linux. --- Python/ceval.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index b76c9ec28119d5..31b81a37464718 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -523,13 +523,6 @@ _Py_InitializeRecursionLimits(PyThreadState *tstate) _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; ts->c_stack_init_base = base; ts->c_stack_init_top = top; - - // Test the stack pointer -#if !defined(NDEBUG) && !defined(__wasi__) - uintptr_t here_addr = _Py_get_machine_stack_pointer(); - assert(ts->c_stack_soft_limit < here_addr); - assert(here_addr < ts->c_stack_top); -#endif } From eab7385858025df9fcb0131f71ec4a46d44e3ae9 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 14 Nov 2025 16:05:42 +0100 Subject: [PATCH 052/638] gh-116146: Avoid empty braces in _testembed.c (GH-141556) --- Programs/_testembed.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 27224e508bdd3e..d0d7d5f03fb9e3 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -2255,7 +2255,7 @@ exec_module(PyObject* self, PyObject* mod) static PyMethodDef create_static_module_methods[] = { {"create_module", create_module, METH_O, NULL}, {"exec_module", exec_module, METH_O, NULL}, - {} + {NULL} }; static struct PyModuleDef create_static_module_def = { From b101e9d36b1aed2bb4bca8aec3e1cc1d1df4f79e Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 14 Nov 2025 15:23:01 +0000 Subject: [PATCH 053/638] Add PyManager troubleshooting steps for direct launch of script files (GH-141530) --- Doc/using/windows.rst | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index e6619b73bd2c26..ee18251919959e 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -4,6 +4,8 @@ .. _Microsoft Store app: https://apps.microsoft.com/detail/9NQ7512CXL7T +.. _legacy launcher: https://www.python.org/ftp/python/3.14.0/win32/launcher.msi + .. _using-on-windows: ************************* @@ -543,12 +545,9 @@ configuration option. The behaviour of shebangs in the Python install manager is subtly different from the previous ``py.exe`` launcher, and the old configuration options no longer apply. If you are specifically reliant on the old behaviour or - configuration, we recommend keeping the legacy launcher. It may be - `downloaded independently `_ - and installed on its own. The legacy launcher's ``py`` command will override - PyManager's one, and you will need to use ``pymanager`` commands for - installing and uninstalling. - + configuration, we recommend installing the `legacy launcher`_. The legacy + launcher's ``py`` command will override PyManager's one by default, and you + will need to use ``pymanager`` commands for installing and uninstalling. .. _Add-AppxPackage: https://learn.microsoft.com/powershell/module/appx/add-appxpackage @@ -859,6 +858,17 @@ default). These scripts are separated for each runtime, and so you may need to add multiple paths. + * - Typing ``script-name.py`` in the terminal opens in a new window. + - This is a known limitation of the operating system. Either specify ``py`` + before the script name, create a batch file containing ``@py "%~dpn0.py" %*`` + with the same name as the script, or install the `legacy launcher`_ + and select it as the association for scripts. + + * - Drag-dropping files onto a script doesn't work + - This is a known limitation of the operating system. It is supported with + the `legacy launcher`_, or with the Python install manager when installed + from the MSI. + .. _windows-embeddable: From da7f4e4b22020cfc6c5b5918756e454ef281848d Mon Sep 17 00:00:00 2001 From: Locked-chess-official <13140752715@163.com> Date: Fri, 14 Nov 2025 23:52:14 +0800 Subject: [PATCH 054/638] gh-141488: Add `Py_` prefix to Include/datetime.h macros (#141493) --- Include/datetime.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Include/datetime.h b/Include/datetime.h index b78cc0e8e2e5ac..ed36e6e48c87d2 100644 --- a/Include/datetime.h +++ b/Include/datetime.h @@ -1,8 +1,8 @@ /* datetime.h */ #ifndef Py_LIMITED_API -#ifndef DATETIME_H -#define DATETIME_H +#ifndef Py_DATETIME_H +#define Py_DATETIME_H #ifdef __cplusplus extern "C" { #endif @@ -263,5 +263,5 @@ static PyDateTime_CAPI *PyDateTimeAPI = NULL; #ifdef __cplusplus } #endif -#endif +#endif /* !Py_DATETIME_H */ #endif /* !Py_LIMITED_API */ From f26ed455d5582a7d66618acf2a93bc4b22a84b47 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Fri, 14 Nov 2025 23:17:59 +0530 Subject: [PATCH 055/638] gh-114203: skip locking if object is already locked by two-mutex critical section (#141476) --- ...-11-14-16-25-15.gh-issue-114203.n3tlQO.rst | 1 + .../test_critical_sections.c | 101 ++++++++++++++++++ Python/critical_section.c | 23 +++- 3 files changed, 120 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-16-25-15.gh-issue-114203.n3tlQO.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-16-25-15.gh-issue-114203.n3tlQO.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-16-25-15.gh-issue-114203.n3tlQO.rst new file mode 100644 index 00000000000000..883f9333cae880 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-16-25-15.gh-issue-114203.n3tlQO.rst @@ -0,0 +1 @@ +Skip locking if object is already locked by two-mutex critical section. diff --git a/Modules/_testinternalcapi/test_critical_sections.c b/Modules/_testinternalcapi/test_critical_sections.c index e0ba37abcdd332..e3b2fe716d48d3 100644 --- a/Modules/_testinternalcapi/test_critical_sections.c +++ b/Modules/_testinternalcapi/test_critical_sections.c @@ -284,10 +284,111 @@ test_critical_sections_gc(PyObject *self, PyObject *Py_UNUSED(args)) #endif +#ifdef Py_GIL_DISABLED + +static PyObject * +test_critical_section1_reacquisition(PyObject *self, PyObject *Py_UNUSED(args)) +{ + PyObject *a = PyDict_New(); + assert(a != NULL); + + PyCriticalSection cs1, cs2; + // First acquisition of critical section on object locks it + PyCriticalSection_Begin(&cs1, a); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert(_PyThreadState_GET()->critical_section == (uintptr_t)&cs1); + // Attempting to re-acquire critical section on same object which + // is already locked by top-most critical section is a no-op. + PyCriticalSection_Begin(&cs2, a); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert(_PyThreadState_GET()->critical_section == (uintptr_t)&cs1); + // Releasing second critical section is a no-op. + PyCriticalSection_End(&cs2); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert(_PyThreadState_GET()->critical_section == (uintptr_t)&cs1); + // Releasing first critical section unlocks the object + PyCriticalSection_End(&cs1); + assert(!PyMutex_IsLocked(&a->ob_mutex)); + + Py_DECREF(a); + Py_RETURN_NONE; +} + +static PyObject * +test_critical_section2_reacquisition(PyObject *self, PyObject *Py_UNUSED(args)) +{ + PyObject *a = PyDict_New(); + assert(a != NULL); + PyObject *b = PyDict_New(); + assert(b != NULL); + + PyCriticalSection2 cs; + // First acquisition of critical section on objects locks them + PyCriticalSection2_Begin(&cs, a, b); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(PyMutex_IsLocked(&b->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert((_PyThreadState_GET()->critical_section & + ~_Py_CRITICAL_SECTION_MASK) == (uintptr_t)&cs); + + // Attempting to re-acquire critical section on either of two + // objects already locked by top-most critical section is a no-op. + + // Check re-acquiring on first object + PyCriticalSection a_cs; + PyCriticalSection_Begin(&a_cs, a); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(PyMutex_IsLocked(&b->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert((_PyThreadState_GET()->critical_section & + ~_Py_CRITICAL_SECTION_MASK) == (uintptr_t)&cs); + // Releasing critical section on either object is a no-op. + PyCriticalSection_End(&a_cs); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(PyMutex_IsLocked(&b->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert((_PyThreadState_GET()->critical_section & + ~_Py_CRITICAL_SECTION_MASK) == (uintptr_t)&cs); + + // Check re-acquiring on second object + PyCriticalSection b_cs; + PyCriticalSection_Begin(&b_cs, b); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(PyMutex_IsLocked(&b->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert((_PyThreadState_GET()->critical_section & + ~_Py_CRITICAL_SECTION_MASK) == (uintptr_t)&cs); + // Releasing critical section on either object is a no-op. + PyCriticalSection_End(&b_cs); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(PyMutex_IsLocked(&b->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert((_PyThreadState_GET()->critical_section & + ~_Py_CRITICAL_SECTION_MASK) == (uintptr_t)&cs); + + // Releasing critical section on both objects unlocks them + PyCriticalSection2_End(&cs); + assert(!PyMutex_IsLocked(&a->ob_mutex)); + assert(!PyMutex_IsLocked(&b->ob_mutex)); + + Py_DECREF(a); + Py_DECREF(b); + Py_RETURN_NONE; +} + +#endif // Py_GIL_DISABLED + static PyMethodDef test_methods[] = { {"test_critical_sections", test_critical_sections, METH_NOARGS}, {"test_critical_sections_nest", test_critical_sections_nest, METH_NOARGS}, {"test_critical_sections_suspend", test_critical_sections_suspend, METH_NOARGS}, +#ifdef Py_GIL_DISABLED + {"test_critical_section1_reacquisition", test_critical_section1_reacquisition, METH_NOARGS}, + {"test_critical_section2_reacquisition", test_critical_section2_reacquisition, METH_NOARGS}, +#endif #ifdef Py_CAN_START_THREADS {"test_critical_sections_threads", test_critical_sections_threads, METH_NOARGS}, {"test_critical_sections_gc", test_critical_sections_gc, METH_NOARGS}, diff --git a/Python/critical_section.c b/Python/critical_section.c index e628ba2f6d19bc..218b580e95176d 100644 --- a/Python/critical_section.c +++ b/Python/critical_section.c @@ -24,11 +24,24 @@ _PyCriticalSection_BeginSlow(PyCriticalSection *c, PyMutex *m) // As an optimisation for locking the same object recursively, skip // locking if the mutex is currently locked by the top-most critical // section. - if (tstate->critical_section && - untag_critical_section(tstate->critical_section)->_cs_mutex == m) { - c->_cs_mutex = NULL; - c->_cs_prev = 0; - return; + // If the top-most critical section is a two-mutex critical section, + // then locking is skipped if either mutex is m. + if (tstate->critical_section) { + PyCriticalSection *prev = untag_critical_section(tstate->critical_section); + if (prev->_cs_mutex == m) { + c->_cs_mutex = NULL; + c->_cs_prev = 0; + return; + } + if (tstate->critical_section & _Py_CRITICAL_SECTION_TWO_MUTEXES) { + PyCriticalSection2 *prev2 = (PyCriticalSection2 *) + untag_critical_section(tstate->critical_section); + if (prev2->_cs_mutex2 == m) { + c->_cs_mutex = NULL; + c->_cs_prev = 0; + return; + } + } } c->_cs_mutex = NULL; c->_cs_prev = (uintptr_t)tstate->critical_section; From 1281be1caf9357ee2a68f7370a88b5cff0110e15 Mon Sep 17 00:00:00 2001 From: Mikhail Efimov Date: Sat, 15 Nov 2025 00:38:39 +0300 Subject: [PATCH 056/638] gh-141367: Use CALL_LIST_APPEND instruction only for lists, not for list subclasses (GH-141398) Co-authored-by: Ken Jin --- Include/internal/pycore_code.h | 4 +-- Lib/test/test_opcache.py | 27 +++++++++++++++++++ ...-11-11-13-40-45.gh-issue-141367.I5KY7F.rst | 2 ++ Python/bytecodes.c | 3 +-- Python/executor_cases.c.h | 4 --- Python/generated_cases.c.h | 7 +---- Python/specialize.c | 17 +++++++----- 7 files changed, 44 insertions(+), 20 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-11-13-40-45.gh-issue-141367.I5KY7F.rst diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 9748e036bf2874..cb9c0aa27a1785 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -311,8 +311,8 @@ PyAPI_FUNC(void) _Py_Specialize_LoadGlobal(PyObject *globals, PyObject *builtins _Py_CODEUNIT *instr, PyObject *name); PyAPI_FUNC(void) _Py_Specialize_StoreSubscr(_PyStackRef container, _PyStackRef sub, _Py_CODEUNIT *instr); -PyAPI_FUNC(void) _Py_Specialize_Call(_PyStackRef callable, _Py_CODEUNIT *instr, - int nargs); +PyAPI_FUNC(void) _Py_Specialize_Call(_PyStackRef callable, _PyStackRef self_or_null, + _Py_CODEUNIT *instr, int nargs); PyAPI_FUNC(void) _Py_Specialize_CallKw(_PyStackRef callable, _Py_CODEUNIT *instr, int nargs); PyAPI_FUNC(void) _Py_Specialize_BinaryOp(_PyStackRef lhs, _PyStackRef rhs, _Py_CODEUNIT *instr, diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index f23f8c053e8431..c7eea75117de8c 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1872,6 +1872,33 @@ def for_iter_generator(): self.assert_specialized(for_iter_generator, "FOR_ITER_GEN") self.assert_no_opcode(for_iter_generator, "FOR_ITER") + @cpython_only + @requires_specialization_ft + def test_call_list_append(self): + # gh-141367: only exact lists should use + # CALL_LIST_APPEND instruction after specialization. + + r = range(_testinternalcapi.SPECIALIZATION_THRESHOLD) + + def list_append(l): + for _ in r: + l.append(1) + + list_append([]) + self.assert_specialized(list_append, "CALL_LIST_APPEND") + self.assert_no_opcode(list_append, "CALL_METHOD_DESCRIPTOR_O") + self.assert_no_opcode(list_append, "CALL") + + def my_list_append(l): + for _ in r: + l.append(1) + + class MyList(list): pass + my_list_append(MyList()) + self.assert_specialized(my_list_append, "CALL_METHOD_DESCRIPTOR_O") + self.assert_no_opcode(my_list_append, "CALL_LIST_APPEND") + self.assert_no_opcode(my_list_append, "CALL") + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-11-13-40-45.gh-issue-141367.I5KY7F.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-11-13-40-45.gh-issue-141367.I5KY7F.rst new file mode 100644 index 00000000000000..cb830fcd9e1270 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-11-13-40-45.gh-issue-141367.I5KY7F.rst @@ -0,0 +1,2 @@ +Specialize ``CALL_LIST_APPEND`` instruction only for lists, not for list +subclasses, to avoid unnecessary deopt. Patch by Mikhail Efimov. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 2c798855a71f55..8a7b784bb9eec2 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3689,7 +3689,7 @@ dummy_func( #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; - _Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + _Py_Specialize_Call(callable, self_or_null, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL); @@ -4395,7 +4395,6 @@ dummy_func( assert(oparg == 1); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); - DEOPT_IF(!PyList_CheckExact(self_o)); DEOPT_IF(!LOCK_OBJECT(self_o)); STAT_INC(CALL, hit); int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 7ba2e9d0d92999..6796abf84ac5f4 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -6037,10 +6037,6 @@ callable = stack_pointer[-3]; assert(oparg == 1); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); - if (!PyList_CheckExact(self_o)) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } if (!LOCK_OBJECT(self_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index a984da6dc912a2..01f65d9dd375f7 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1533,7 +1533,7 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + _Py_Specialize_Call(callable, self_or_null, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); stack_pointer = _PyFrame_GetStackPointer(frame); DISPATCH_SAME_OPARG(); } @@ -3470,11 +3470,6 @@ self = nos; assert(oparg == 1); PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); - if (!PyList_CheckExact(self_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } if (!LOCK_OBJECT(self_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); diff --git a/Python/specialize.c b/Python/specialize.c index 2193596a331d3c..19433bc7a74319 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1602,8 +1602,8 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) } static int -specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, - int nargs) +specialize_method_descriptor(PyMethodDescrObject *descr, PyObject *self_or_null, + _Py_CODEUNIT *instr, int nargs) { switch (descr->d_method->ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | @@ -1627,8 +1627,11 @@ specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, bool pop = (next.op.code == POP_TOP); int oparg = instr->op.arg; if ((PyObject *)descr == list_append && oparg == 1 && pop) { - specialize(instr, CALL_LIST_APPEND); - return 0; + assert(self_or_null != NULL); + if (PyList_CheckExact(self_or_null)) { + specialize(instr, CALL_LIST_APPEND); + return 0; + } } specialize(instr, CALL_METHOD_DESCRIPTOR_O); return 0; @@ -1766,7 +1769,7 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) } Py_NO_INLINE void -_Py_Specialize_Call(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) +_Py_Specialize_Call(_PyStackRef callable_st, _PyStackRef self_or_null_st, _Py_CODEUNIT *instr, int nargs) { PyObject *callable = PyStackRef_AsPyObjectBorrow(callable_st); @@ -1784,7 +1787,9 @@ _Py_Specialize_Call(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) fail = specialize_class_call(callable, instr, nargs); } else if (Py_IS_TYPE(callable, &PyMethodDescr_Type)) { - fail = specialize_method_descriptor((PyMethodDescrObject *)callable, instr, nargs); + PyObject *self_or_null = PyStackRef_AsPyObjectBorrow(self_or_null_st); + fail = specialize_method_descriptor((PyMethodDescrObject *)callable, + self_or_null, instr, nargs); } else if (PyMethod_Check(callable)) { PyObject *func = ((PyMethodObject *)callable)->im_func; From f0a8bc737ab2f04d4196eee154cb1e17e26ad585 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 14 Nov 2025 17:25:45 -0600 Subject: [PATCH 057/638] gh-140938: Raise ValueError for infinite inputs to stdev/pstdev (GH-141531) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Raise ValueError for infinite inputs to stdev/pstdev --- Co-authored-by: Gregory P. Smith <68491+gpshead@users.noreply.github.com> Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/statistics.py | 18 ++++++++++++++---- Lib/test/test_statistics.py | 9 ++++++++- ...5-11-13-14-51-30.gh-issue-140938.kXsHHv.rst | 2 ++ 3 files changed, 24 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-13-14-51-30.gh-issue-140938.kXsHHv.rst diff --git a/Lib/statistics.py b/Lib/statistics.py index 3d805cb073987d..26cf925529ea60 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -619,9 +619,14 @@ def stdev(data, xbar=None): if n < 2: raise StatisticsError('stdev requires at least two data points') mss = ss / (n - 1) + try: + mss_numerator = mss.numerator + mss_denominator = mss.denominator + except AttributeError: + raise ValueError('inf or nan encountered in data') if issubclass(T, Decimal): - return _decimal_sqrt_of_frac(mss.numerator, mss.denominator) - return _float_sqrt_of_frac(mss.numerator, mss.denominator) + return _decimal_sqrt_of_frac(mss_numerator, mss_denominator) + return _float_sqrt_of_frac(mss_numerator, mss_denominator) def pstdev(data, mu=None): @@ -637,9 +642,14 @@ def pstdev(data, mu=None): if n < 1: raise StatisticsError('pstdev requires at least one data point') mss = ss / n + try: + mss_numerator = mss.numerator + mss_denominator = mss.denominator + except AttributeError: + raise ValueError('inf or nan encountered in data') if issubclass(T, Decimal): - return _decimal_sqrt_of_frac(mss.numerator, mss.denominator) - return _float_sqrt_of_frac(mss.numerator, mss.denominator) + return _decimal_sqrt_of_frac(mss_numerator, mss_denominator) + return _float_sqrt_of_frac(mss_numerator, mss_denominator) ## Statistics for relations between two inputs ############################# diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index 8250b0aef09aec..677a87b51b9192 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -2005,7 +2005,6 @@ def test_iter_list_same(self): expected = self.func(data) self.assertEqual(self.func(iter(data)), expected) - class TestPVariance(VarianceStdevMixin, NumericTestCase, UnivariateTypeMixin): # Tests for population variance. def setUp(self): @@ -2113,6 +2112,14 @@ def test_center_not_at_mean(self): self.assertEqual(self.func(data), 2.5) self.assertEqual(self.func(data, mu=0.5), 6.5) + def test_gh_140938(self): + # Inputs with inf/nan should raise a ValueError + with self.assertRaises(ValueError): + self.func([1.0, math.inf]) + with self.assertRaises(ValueError): + self.func([1.0, math.nan]) + + class TestSqrtHelpers(unittest.TestCase): def test_integer_sqrt_of_frac_rto(self): diff --git a/Misc/NEWS.d/next/Library/2025-11-13-14-51-30.gh-issue-140938.kXsHHv.rst b/Misc/NEWS.d/next/Library/2025-11-13-14-51-30.gh-issue-140938.kXsHHv.rst new file mode 100644 index 00000000000000..bd3044002a2d54 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-13-14-51-30.gh-issue-140938.kXsHHv.rst @@ -0,0 +1,2 @@ +The :func:`statistics.stdev` and :func:`statistics.pstdev` functions now raise a +:exc:`ValueError` when the input contains an infinity or a NaN. From 453d886f8592d2f4346d5621b1e4ff31c24338d5 Mon Sep 17 00:00:00 2001 From: Guo Ci Date: Fri, 14 Nov 2025 19:13:37 -0500 Subject: [PATCH 058/638] GH-90344: replace single-call `io.IncrementalNewlineDecoder` usage with non-incremental newline decoders (GH-30276) Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Brett Cannon --- Lib/doctest.py | 9 ++------- Lib/importlib/_bootstrap_external.py | 3 +-- Lib/test/test_importlib/test_abc.py | 2 +- .../2025-10-31-14-03-42.gh-issue-90344.gvZigO.rst | 1 + 4 files changed, 5 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-31-14-03-42.gh-issue-90344.gvZigO.rst diff --git a/Lib/doctest.py b/Lib/doctest.py index 92a2ab4f7e66f8..ad8fb900f692c7 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -104,7 +104,7 @@ def _test(): import traceback import types import unittest -from io import StringIO, IncrementalNewlineDecoder +from io import StringIO, TextIOWrapper, BytesIO from collections import namedtuple import _colorize # Used in doctests from _colorize import ANSIColors, can_colorize @@ -237,10 +237,6 @@ def _normalize_module(module, depth=2): else: raise TypeError("Expected a module, string, or None") -def _newline_convert(data): - # The IO module provides a handy decoder for universal newline conversion - return IncrementalNewlineDecoder(None, True).decode(data, True) - def _load_testfile(filename, package, module_relative, encoding): if module_relative: package = _normalize_module(package, 3) @@ -252,10 +248,9 @@ def _load_testfile(filename, package, module_relative, encoding): pass if hasattr(loader, 'get_data'): file_contents = loader.get_data(filename) - file_contents = file_contents.decode(encoding) # get_data() opens files as 'rb', so one must do the equivalent # conversion as universal newlines would do. - return _newline_convert(file_contents), filename + return TextIOWrapper(BytesIO(file_contents), encoding=encoding, newline=None).read(), filename with open(filename, encoding=encoding) as f: return f.read(), filename diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 4ab0e79ea6efeb..192c0261408ead 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -552,8 +552,7 @@ def decode_source(source_bytes): import tokenize # To avoid bootstrap issues. source_bytes_readline = _io.BytesIO(source_bytes).readline encoding = tokenize.detect_encoding(source_bytes_readline) - newline_decoder = _io.IncrementalNewlineDecoder(None, True) - return newline_decoder.decode(source_bytes.decode(encoding[0])) + return _io.TextIOWrapper(_io.BytesIO(source_bytes), encoding=encoding[0], newline=None).read() # Module specifications ####################################################### diff --git a/Lib/test/test_importlib/test_abc.py b/Lib/test/test_importlib/test_abc.py index dd943210ffca3c..bd1540ce403ce2 100644 --- a/Lib/test/test_importlib/test_abc.py +++ b/Lib/test/test_importlib/test_abc.py @@ -904,7 +904,7 @@ def test_universal_newlines(self): mock = self.SourceOnlyLoaderMock('mod.file') source = "x = 42\r\ny = -13\r\n" mock.source = source.encode('utf-8') - expect = io.IncrementalNewlineDecoder(None, True).decode(source) + expect = io.StringIO(source, newline=None).getvalue() self.assertEqual(mock.get_source(name), expect) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-31-14-03-42.gh-issue-90344.gvZigO.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-31-14-03-42.gh-issue-90344.gvZigO.rst new file mode 100644 index 00000000000000..b1d05354f65c71 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-31-14-03-42.gh-issue-90344.gvZigO.rst @@ -0,0 +1 @@ +Replace :class:`io.IncrementalNewlineDecoder` with non incremental newline decoders in codebase where :meth:`!io.IncrementalNewlineDecoder.decode` was being called once. From 53d65c840e038ce9a5782fbd3da963c7aba90570 Mon Sep 17 00:00:00 2001 From: Takuya UESHIN Date: Fri, 14 Nov 2025 16:59:51 -0800 Subject: [PATCH 059/638] gh-136442: Fix unittest to return exit code 5 when setUpClass raises an exception (#136487) --- Lib/test/test_unittest/test_program.py | 20 +++++++++++++++++++ Lib/unittest/main.py | 8 ++++---- ...-07-09-21-45-51.gh-issue-136442.jlbklP.rst | 1 + 3 files changed, 25 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2025-07-09-21-45-51.gh-issue-136442.jlbklP.rst diff --git a/Lib/test/test_unittest/test_program.py b/Lib/test/test_unittest/test_program.py index 6092ed292d8f60..8ed92373e5e984 100644 --- a/Lib/test/test_unittest/test_program.py +++ b/Lib/test/test_unittest/test_program.py @@ -75,6 +75,14 @@ def testUnexpectedSuccess(self): class Empty(unittest.TestCase): pass + class SetUpClassFailure(unittest.TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + raise Exception + def testPass(self): + pass + class TestLoader(unittest.TestLoader): """Test loader that returns a suite containing the supplied testcase.""" @@ -191,6 +199,18 @@ def test_ExitEmptySuite(self): out = stream.getvalue() self.assertIn('\nNO TESTS RAN\n', out) + def test_ExitSetUpClassFailureSuite(self): + stream = BufferedWriter() + with self.assertRaises(SystemExit) as cm: + unittest.main( + argv=["setup_class_failure"], + testRunner=unittest.TextTestRunner(stream=stream), + testLoader=self.TestLoader(self.SetUpClassFailure)) + self.assertEqual(cm.exception.code, 1) + out = stream.getvalue() + self.assertIn("ERROR: setUpClass", out) + self.assertIn("SetUpClassFailure", out) + class InitialisableProgram(unittest.TestProgram): exit = False diff --git a/Lib/unittest/main.py b/Lib/unittest/main.py index 6fd949581f3146..be99d93c78cca6 100644 --- a/Lib/unittest/main.py +++ b/Lib/unittest/main.py @@ -269,12 +269,12 @@ def runTests(self): testRunner = self.testRunner self.result = testRunner.run(self.test) if self.exit: - if self.result.testsRun == 0 and len(self.result.skipped) == 0: + if not self.result.wasSuccessful(): + sys.exit(1) + elif self.result.testsRun == 0 and len(self.result.skipped) == 0: sys.exit(_NO_TESTS_EXITCODE) - elif self.result.wasSuccessful(): - sys.exit(0) else: - sys.exit(1) + sys.exit(0) main = TestProgram diff --git a/Misc/NEWS.d/next/Tests/2025-07-09-21-45-51.gh-issue-136442.jlbklP.rst b/Misc/NEWS.d/next/Tests/2025-07-09-21-45-51.gh-issue-136442.jlbklP.rst new file mode 100644 index 00000000000000..f87fb1113cad12 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2025-07-09-21-45-51.gh-issue-136442.jlbklP.rst @@ -0,0 +1 @@ +Use exitcode ``1`` instead of ``5`` if :func:`unittest.TestCase.setUpClass` raises an exception From 4ceb077c5cea30fef734f4c4e92c18d978be6c38 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sat, 15 Nov 2025 02:23:54 +0000 Subject: [PATCH 060/638] gh-141579: Fix perf_jit backend in sys.activate_stack_trampoline() (#141580) --- Lib/test/test_perf_profiler.py | 18 ++++++++++++++++++ ...5-11-15-01-21-00.gh-issue-141579.aB7cD9.rst | 2 ++ Python/sysmodule.c | 16 ++++++++-------- 3 files changed, 28 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-01-21-00.gh-issue-141579.aB7cD9.rst diff --git a/Lib/test/test_perf_profiler.py b/Lib/test/test_perf_profiler.py index 13424991639215..e6852c93e69830 100644 --- a/Lib/test/test_perf_profiler.py +++ b/Lib/test/test_perf_profiler.py @@ -238,6 +238,24 @@ def test_sys_api_get_status(self): """ assert_python_ok("-c", code, PYTHON_JIT="0") + def test_sys_api_perf_jit_backend(self): + code = """if 1: + import sys + sys.activate_stack_trampoline("perf_jit") + assert sys.is_stack_trampoline_active() is True + sys.deactivate_stack_trampoline() + assert sys.is_stack_trampoline_active() is False + """ + assert_python_ok("-c", code, PYTHON_JIT="0") + + def test_sys_api_with_existing_perf_jit_trampoline(self): + code = """if 1: + import sys + sys.activate_stack_trampoline("perf_jit") + sys.activate_stack_trampoline("perf_jit") + """ + assert_python_ok("-c", code, PYTHON_JIT="0") + def is_unwinding_reliable_with_frame_pointers(): cflags = sysconfig.get_config_var("PY_CORE_CFLAGS") diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-01-21-00.gh-issue-141579.aB7cD9.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-01-21-00.gh-issue-141579.aB7cD9.rst new file mode 100644 index 00000000000000..8ab9979c39917b --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-01-21-00.gh-issue-141579.aB7cD9.rst @@ -0,0 +1,2 @@ +Fix :func:`sys.activate_stack_trampoline` to properly support the +``perf_jit`` backend. Patch by Pablo Galindo. diff --git a/Python/sysmodule.c b/Python/sysmodule.c index a611844f76e090..b4b441bf4d9519 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2380,14 +2380,14 @@ sys_activate_stack_trampoline_impl(PyObject *module, const char *backend) return NULL; } } - else if (strcmp(backend, "perf_jit") == 0) { - _PyPerf_Callbacks cur_cb; - _PyPerfTrampoline_GetCallbacks(&cur_cb); - if (cur_cb.write_state != _Py_perfmap_jit_callbacks.write_state) { - if (_PyPerfTrampoline_SetCallbacks(&_Py_perfmap_jit_callbacks) < 0 ) { - PyErr_SetString(PyExc_ValueError, "can't activate perf jit trampoline"); - return NULL; - } + } + else if (strcmp(backend, "perf_jit") == 0) { + _PyPerf_Callbacks cur_cb; + _PyPerfTrampoline_GetCallbacks(&cur_cb); + if (cur_cb.write_state != _Py_perfmap_jit_callbacks.write_state) { + if (_PyPerfTrampoline_SetCallbacks(&_Py_perfmap_jit_callbacks) < 0 ) { + PyErr_SetString(PyExc_ValueError, "can't activate perf jit trampoline"); + return NULL; } } } From ed81baf81f144e14510c492b71cf860472b0a0b7 Mon Sep 17 00:00:00 2001 From: Yongzi Li <204532581+Yzi-Li@users.noreply.github.com> Date: Sun, 16 Nov 2025 00:14:23 +0800 Subject: [PATCH 061/638] gh-140458: `xmlrpc.client` raises Fault, does not returns it. (GH-140759) --- Doc/library/xmlrpc.client.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/xmlrpc.client.rst b/Doc/library/xmlrpc.client.rst index a21c7d3e4e3ad5..e4912629aac6e0 100644 --- a/Doc/library/xmlrpc.client.rst +++ b/Doc/library/xmlrpc.client.rst @@ -179,9 +179,9 @@ ServerProxy Objects A :class:`ServerProxy` instance has a method corresponding to each remote procedure call accepted by the XML-RPC server. Calling the method performs an RPC, dispatched by both name and argument signature (e.g. the same method name -can be overloaded with multiple argument signatures). The RPC finishes by -returning a value, which may be either returned data in a conformant type or a -:class:`Fault` or :class:`ProtocolError` object indicating an error. +can be overloaded with multiple argument signatures). The RPC finishes either +by returning data in a conformant type or by raising a :class:`Fault` or +:class:`ProtocolError` exception indicating an error. Servers that support the XML introspection API support some common methods grouped under the reserved :attr:`~ServerProxy.system` attribute: From 85f3009d7504ddcc01de715c494067e89c16303c Mon Sep 17 00:00:00 2001 From: Shamil Date: Sat, 15 Nov 2025 20:46:54 +0300 Subject: [PATCH 062/638] gh-141553: Fix incorrect function signatures in `_testmultiphase` (#141554) --- Modules/_testmultiphase.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c index 220fa888e49a52..cd2d7b65598277 100644 --- a/Modules/_testmultiphase.c +++ b/Modules/_testmultiphase.c @@ -1061,7 +1061,7 @@ PyModInit__test_from_modexport_exception(void) } static PyObject * -modexport_create_string(PyObject *spec, PyObject *def) +modexport_create_string(PyObject *spec, PyModuleDef *def) { assert(def == NULL); return PyUnicode_FromString("is this \xf0\x9f\xa6\x8b... a module?"); @@ -1138,8 +1138,9 @@ modexport_get_empty_slots(PyObject *mod, PyObject *arg) } static void -modexport_smoke_free(PyObject *mod) +modexport_smoke_free(void *op) { + PyObject *mod = (PyObject *)op; int *state = PyModule_GetState(mod); if (!state) { PyErr_FormatUnraisable("Exception ignored in module %R free", mod); From ed73c909f278a1eb558b120ef8ed2c0f8528bf58 Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Sun, 16 Nov 2025 04:19:41 +0800 Subject: [PATCH 063/638] gh-139109: JIT _EXIT_TRACE to ENTER_EXECUTOR rather than _DEOPT (GH-141573) --- Include/internal/pycore_optimizer.h | 2 +- Lib/test/test_capi/test_opt.py | 36 +++++++++++++++++++++++++++++ Python/bytecodes.c | 4 ++-- Python/generated_cases.c.h | 4 ++-- Python/optimizer.c | 6 ++--- 5 files changed, 44 insertions(+), 8 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 653285a2c6b79b..0307a174e77346 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -362,7 +362,7 @@ PyAPI_FUNC(int) _PyDumpExecutors(FILE *out); extern void _Py_ClearExecutorDeletionList(PyInterpreterState *interp); #endif -int _PyJit_translate_single_bytecode_to_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, bool stop_tracing); +int _PyJit_translate_single_bytecode_to_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, int stop_tracing_opcode); int _PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index f06c6cbda2976c..25372fee58e0d7 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -40,6 +40,17 @@ def get_first_executor(func): pass return None +def get_all_executors(func): + code = func.__code__ + co_code = code.co_code + executors = [] + for i in range(0, len(co_code), 2): + try: + executors.append(_opcode.get_executor(code, i)) + except ValueError: + pass + return executors + def iter_opnames(ex): for item in ex: @@ -2629,6 +2640,31 @@ def gen(): next(g) """ % _testinternalcapi.SPECIALIZATION_THRESHOLD)) + def test_executor_side_exits_create_another_executor(self): + def f(): + for x in range(TIER2_THRESHOLD + 3): + for y in range(TIER2_THRESHOLD + 3): + z = x + y + + f() + all_executors = get_all_executors(f) + # Inner loop warms up first. + # Outer loop warms up later, linking to the inner one. + # Therefore, we have at least two executors. + self.assertGreaterEqual(len(all_executors), 2) + for executor in all_executors: + opnames = list(get_opnames(executor)) + # Assert all executors first terminator ends in + # _EXIT_TRACE or _JUMP_TO_TOP, not _DEOPT + for idx, op in enumerate(opnames): + if op == "_EXIT_TRACE" or op == "_JUMP_TO_TOP": + break + elif op == "_DEOPT": + self.fail(f"_DEOPT encountered first at executor" + f" {executor} at offset {idx} rather" + f" than expected _EXIT_TRACE") + + def global_identity(x): return x diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 8a7b784bb9eec2..565eaa7a599175 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5643,7 +5643,7 @@ dummy_func( bool stop_tracing = (opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO || opcode == INTERPRETER_EXIT); - int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing); + int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing ? _DEOPT : 0); if (full) { LEAVE_TRACING(); int err = stop_tracing_and_jit(tstate, frame); @@ -5683,7 +5683,7 @@ dummy_func( #if _Py_TIER2 assert(IS_JIT_TRACING()); int opcode = next_instr->op.code; - _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL, true); + _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL, _EXIT_TRACE); LEAVE_TRACING(); int err = stop_tracing_and_jit(tstate, frame); ERROR_IF(err < 0); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 01f65d9dd375f7..0d4678df68ce2d 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -12263,7 +12263,7 @@ JUMP_TO_LABEL(error); opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO || opcode == INTERPRETER_EXIT); _PyFrame_SetStackPointer(frame, stack_pointer); - int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing); + int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing ? _DEOPT : 0); stack_pointer = _PyFrame_GetStackPointer(frame); if (full) { LEAVE_TRACING(); @@ -12309,7 +12309,7 @@ JUMP_TO_LABEL(error); assert(IS_JIT_TRACING()); int opcode = next_instr->op.code; _PyFrame_SetStackPointer(frame, stack_pointer); - _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL, true); + _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL, _EXIT_TRACE); stack_pointer = _PyFrame_GetStackPointer(frame); LEAVE_TRACING(); _PyFrame_SetStackPointer(frame, stack_pointer); diff --git a/Python/optimizer.c b/Python/optimizer.c index 65007a256d0c3b..9db894f0bf054a 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -574,7 +574,7 @@ _PyJit_translate_single_bytecode_to_trace( PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, - bool stop_tracing) + int stop_tracing_opcode) { #ifdef Py_DEBUG @@ -637,8 +637,8 @@ _PyJit_translate_single_bytecode_to_trace( goto full; } - if (stop_tracing) { - ADD_TO_TRACE(_DEOPT, 0, 0, target); + if (stop_tracing_opcode != 0) { + ADD_TO_TRACE(stop_tracing_opcode, 0, 0, target); goto done; } From e33afa7ddbca3fca38f4ec4369b620c37cb092e2 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Sun, 16 Nov 2025 18:50:54 +0000 Subject: [PATCH 064/638] gh-141004: Document the `PyPickleBuffer_*` C API (GH-141630) Co-authored-by: Peter Bierma --- Doc/c-api/concrete.rst | 1 + Doc/c-api/picklebuffer.rst | 59 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 Doc/c-api/picklebuffer.rst diff --git a/Doc/c-api/concrete.rst b/Doc/c-api/concrete.rst index a5c5a53236c9a4..1746fe95eaaca9 100644 --- a/Doc/c-api/concrete.rst +++ b/Doc/c-api/concrete.rst @@ -109,6 +109,7 @@ Other Objects descriptor.rst slice.rst memoryview.rst + picklebuffer.rst weakref.rst capsule.rst frame.rst diff --git a/Doc/c-api/picklebuffer.rst b/Doc/c-api/picklebuffer.rst new file mode 100644 index 00000000000000..9e2d92341b0f93 --- /dev/null +++ b/Doc/c-api/picklebuffer.rst @@ -0,0 +1,59 @@ +.. highlight:: c + +.. _picklebuffer-objects: + +.. index:: + pair: object; PickleBuffer + +Pickle buffer objects +--------------------- + +.. versionadded:: 3.8 + +A :class:`pickle.PickleBuffer` object wraps a :ref:`buffer-providing object +` for out-of-band data transfer with the :mod:`pickle` module. + + +.. c:var:: PyTypeObject PyPickleBuffer_Type + + This instance of :c:type:`PyTypeObject` represents the Python pickle buffer type. + This is the same object as :class:`pickle.PickleBuffer` in the Python layer. + + +.. c:function:: int PyPickleBuffer_Check(PyObject *op) + + Return true if *op* is a pickle buffer instance. + This function always succeeds. + + +.. c:function:: PyObject *PyPickleBuffer_FromObject(PyObject *obj) + + Create a pickle buffer from the object *obj*. + + This function will fail if *obj* doesn't support the :ref:`buffer protocol `. + + On success, return a new pickle buffer instance. + On failure, set an exception and return ``NULL``. + + Analogous to calling :class:`pickle.PickleBuffer` with *obj* in Python. + + +.. c:function:: const Py_buffer *PyPickleBuffer_GetBuffer(PyObject *picklebuf) + + Get a pointer to the underlying :c:type:`Py_buffer` that the pickle buffer wraps. + + The returned pointer is valid as long as *picklebuf* is alive and has not been + released. The caller must not modify or free the returned :c:type:`Py_buffer`. + If the pickle buffer has been released, raise :exc:`ValueError`. + + On success, return a pointer to the buffer view. + On failure, set an exception and return ``NULL``. + + +.. c:function:: int PyPickleBuffer_Release(PyObject *picklebuf) + + Release the underlying buffer held by the pickle buffer. + + Return ``0`` on success. On failure, set an exception and return ``-1``. + + Analogous to calling :meth:`pickle.PickleBuffer.release` in Python. From 5348c200f5b26d6dd21d900b2b4cb684150d4b01 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sun, 16 Nov 2025 10:58:28 -0800 Subject: [PATCH 065/638] gh-125115 : Refactor the pdb parsing issue so positional arguments can pass through (#140933) --- Lib/pdb.py | 78 ++++++++++--------- Lib/test/test_pdb.py | 5 +- ...-11-03-05-38-31.gh-issue-125115.jGS8MN.rst | 1 + 3 files changed, 48 insertions(+), 36 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-03-05-38-31.gh-issue-125115.jGS8MN.rst diff --git a/Lib/pdb.py b/Lib/pdb.py index b799a113503502..76bb28d7396452 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -3548,7 +3548,15 @@ def exit_with_permission_help_text(): sys.exit(1) -def main(): +def parse_args(): + # We want pdb to be as intuitive as possible to users, so we need to do some + # heuristic parsing to deal with ambiguity. + # For example: + # "python -m pdb -m foo -p 1" should pass "-p 1" to "foo". + # "python -m pdb foo.py -m bar" should pass "-m bar" to "foo.py". + # "python -m pdb -m foo -m bar" should pass "-m bar" to "foo". + # This require some customized parsing logic to find the actual debug target. + import argparse parser = argparse.ArgumentParser( @@ -3559,28 +3567,48 @@ def main(): color=True, ) - # We need to maunally get the script from args, because the first positional - # arguments could be either the script we need to debug, or the argument - # to the -m module + # Get all the commands out first. For backwards compatibility, we allow + # -c commands to be after the target. parser.add_argument('-c', '--command', action='append', default=[], metavar='command', dest='commands', help='pdb commands to execute as if given in a .pdbrc file') - parser.add_argument('-m', metavar='module', dest='module') - parser.add_argument('-p', '--pid', type=int, help="attach to the specified PID", default=None) - if len(sys.argv) == 1: + opts, args = parser.parse_known_args() + + if not args: # If no arguments were given (python -m pdb), print the whole help message. # Without this check, argparse would only complain about missing required arguments. + # We need to add the arguments definitions here to get a proper help message. + parser.add_argument('-m', metavar='module', dest='module') + parser.add_argument('-p', '--pid', type=int, help="attach to the specified PID", default=None) parser.print_help() sys.exit(2) + elif args[0] == '-p' or args[0] == '--pid': + # Attach to a pid + parser.add_argument('-p', '--pid', type=int, help="attach to the specified PID", default=None) + opts, args = parser.parse_known_args() + if args: + # For --pid, any extra arguments are invalid. + parser.error(f"unrecognized arguments: {' '.join(args)}") + elif args[0] == '-m': + # Debug a module, we only need the first -m module argument. + # The rest is passed to the module itself. + parser.add_argument('-m', metavar='module', dest='module') + opt_module = parser.parse_args(args[:2]) + opts.module = opt_module.module + args = args[2:] + elif args[0].startswith('-'): + # Invalid argument before the script name. + invalid_args = list(itertools.takewhile(lambda a: a.startswith('-'), args)) + parser.error(f"unrecognized arguments: {' '.join(invalid_args)}") - opts, args = parser.parse_known_args() + # Otherwise it's debugging a script and we already parsed all -c commands. + + return opts, args - if opts.pid: - # If attaching to a remote pid, unrecognized arguments are not allowed. - # This will raise an error if there are extra unrecognized arguments. - opts = parser.parse_args() - if opts.module: - parser.error("argument -m: not allowed with argument --pid") +def main(): + opts, args = parse_args() + + if getattr(opts, 'pid', None) is not None: try: attach(opts.pid, opts.commands) except RuntimeError: @@ -3592,30 +3620,10 @@ def main(): except PermissionError: exit_with_permission_help_text() return - elif opts.module: - # If a module is being debugged, we consider the arguments after "-m module" to - # be potential arguments to the module itself. We need to parse the arguments - # before "-m" to check if there is any invalid argument. - # e.g. "python -m pdb -m foo --spam" means passing "--spam" to "foo" - # "python -m pdb --spam -m foo" means passing "--spam" to "pdb" and is invalid - idx = sys.argv.index('-m') - args_to_pdb = sys.argv[1:idx] - # This will raise an error if there are invalid arguments - parser.parse_args(args_to_pdb) - else: - # If a script is being debugged, then pdb expects the script name as the first argument. - # Anything before the script is considered an argument to pdb itself, which would - # be invalid because it's not parsed by argparse. - invalid_args = list(itertools.takewhile(lambda a: a.startswith('-'), args)) - if invalid_args: - parser.error(f"unrecognized arguments: {' '.join(invalid_args)}") - - if opts.module: + elif getattr(opts, 'module', None) is not None: file = opts.module target = _ModuleTarget(file) else: - if not args: - parser.error("no module or script to run") file = args.pop(0) if file.endswith('.pyz'): target = _ZipTarget(file) diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 9a7d855003551a..2ca689e0adf710 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -3974,7 +3974,10 @@ def test_run_module_with_args(self): commands = """ continue """ - self._run_pdb(["calendar", "-m"], commands, expected_returncode=2) + self._run_pdb(["calendar", "-m"], commands, expected_returncode=1) + + _, stderr = self._run_pdb(["-m", "calendar", "-p", "1"], commands) + self.assertIn("unrecognized arguments: -p", stderr) stdout, _ = self._run_pdb(["-m", "calendar", "1"], commands) self.assertIn("December", stdout) diff --git a/Misc/NEWS.d/next/Library/2025-11-03-05-38-31.gh-issue-125115.jGS8MN.rst b/Misc/NEWS.d/next/Library/2025-11-03-05-38-31.gh-issue-125115.jGS8MN.rst new file mode 100644 index 00000000000000..d36debec3ed6cc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-03-05-38-31.gh-issue-125115.jGS8MN.rst @@ -0,0 +1 @@ +Refactor the :mod:`pdb` parsing issue so positional arguments can pass through intuitively. From be699d6c7c8793d3eb464f2e5d3f10262fe3bc37 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sun, 16 Nov 2025 14:25:50 -0500 Subject: [PATCH 066/638] gh-141004: Document missing `PyCFunction*` and `PyCMethod*` APIs (GH-141253) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Doc/c-api/structures.rst | 93 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 58dd915e04f619..414dfdc84e61c9 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -447,6 +447,25 @@ definition with the same method name. slot. This is helpful because calls to PyCFunctions are optimized more than wrapper object calls. + +.. c:var:: PyTypeObject PyCMethod_Type + + The type object corresponding to Python C method objects. This is + available as :class:`types.BuiltinMethodType` in the Python layer. + + +.. c:function:: int PyCMethod_Check(PyObject *op) + + Return true if *op* is an instance of the :c:type:`PyCMethod_Type` type + or a subtype of it. This function always succeeds. + + +.. c:function:: int PyCMethod_CheckExact(PyObject *op) + + This is the same as :c:func:`PyCMethod_Check`, but does not account for + subtypes. + + .. c:function:: PyObject * PyCMethod_New(PyMethodDef *ml, PyObject *self, PyObject *module, PyTypeObject *cls) Turn *ml* into a Python :term:`callable` object. @@ -472,6 +491,24 @@ definition with the same method name. .. versionadded:: 3.9 +.. c:var:: PyTypeObject PyCFunction_Type + + The type object corresponding to Python C function objects. This is + available as :class:`types.BuiltinFunctionType` in the Python layer. + + +.. c:function:: int PyCFunction_Check(PyObject *op) + + Return true if *op* is an instance of the :c:type:`PyCFunction_Type` type + or a subtype of it. This function always succeeds. + + +.. c:function:: int PyCFunction_CheckExact(PyObject *op) + + This is the same as :c:func:`PyCFunction_Check`, but does not account for + subtypes. + + .. c:function:: PyObject * PyCFunction_NewEx(PyMethodDef *ml, PyObject *self, PyObject *module) Equivalent to ``PyCMethod_New(ml, self, module, NULL)``. @@ -482,6 +519,62 @@ definition with the same method name. Equivalent to ``PyCMethod_New(ml, self, NULL, NULL)``. +.. c:function:: int PyCFunction_GetFlags(PyObject *func) + + Get the function's flags on *func* as they were passed to + :c:member:`~PyMethodDef.ml_flags`. + + If *func* is not a C function object, this fails with an exception. + *func* must not be ``NULL``. + + This function returns the function's flags on success, and ``-1`` with an + exception set on failure. + + +.. c:function:: int PyCFunction_GET_FLAGS(PyObject *func) + + This is the same as :c:func:`PyCFunction_GetFlags`, but without error + or type checking. + + +.. c:function:: PyCFunction PyCFunction_GetFunction(PyObject *func) + + Get the function pointer on *func* as it was passed to + :c:member:`~PyMethodDef.ml_meth`. + + If *func* is not a C function object, this fails with an exception. + *func* must not be ``NULL``. + + This function returns the function pointer on success, and ``NULL`` with an + exception set on failure. + + +.. c:function:: int PyCFunction_GET_FUNCTION(PyObject *func) + + This is the same as :c:func:`PyCFunction_GetFunction`, but without error + or type checking. + + +.. c:function:: PyObject *PyCFunction_GetSelf(PyObject *func) + + Get the "self" object on *func*. This is the object that would be passed + to the first argument of a :c:type:`PyCFunction`. For C function objects + created through a :c:type:`PyMethodDef` on a :c:type:`PyModuleDef`, this + is the resulting module object. + + If *func* is not a C function object, this fails with an exception. + *func* must not be ``NULL``. + + This function returns a :term:`borrowed reference` to the "self" object + on success, and ``NULL`` with an exception set on failure. + + +.. c:function:: PyObject *PyCFunction_GET_SELF(PyObject *func) + + This is the same as :c:func:`PyCFunction_GetSelf`, but without error or + type checking. + + Accessing attributes of extension types --------------------------------------- From 8be3b2f479431f670f2e81e41b52e698c0806289 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sun, 16 Nov 2025 13:57:07 -0800 Subject: [PATCH 067/638] gh-136057: Allow step and next to step over for loops (#136160) --- Lib/bdb.py | 22 ++++++++++--- Lib/test/test_pdb.py | 31 +++++++++++++++++++ ...-07-01-04-57-57.gh-issue-136057.4-t596.rst | 1 + 3 files changed, 50 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-07-01-04-57-57.gh-issue-136057.4-t596.rst diff --git a/Lib/bdb.py b/Lib/bdb.py index efc3e0a235ac8e..50cf2b3f5b3e45 100644 --- a/Lib/bdb.py +++ b/Lib/bdb.py @@ -199,6 +199,8 @@ def __init__(self, skip=None, backend='settrace'): self.frame_returning = None self.trace_opcodes = False self.enterframe = None + self.cmdframe = None + self.cmdlineno = None self.code_linenos = weakref.WeakKeyDictionary() self.backend = backend if backend == 'monitoring': @@ -297,7 +299,12 @@ def dispatch_line(self, frame): self.user_line(). Raise BdbQuit if self.quitting is set. Return self.trace_dispatch to continue tracing in this scope. """ - if self.stop_here(frame) or self.break_here(frame): + # GH-136057 + # For line events, we don't want to stop at the same line where + # the latest next/step command was issued. + if (self.stop_here(frame) or self.break_here(frame)) and not ( + self.cmdframe == frame and self.cmdlineno == frame.f_lineno + ): self.user_line(frame) self.restart_events() if self.quitting: raise BdbQuit @@ -526,7 +533,8 @@ def _set_trace_opcodes(self, trace_opcodes): if self.monitoring_tracer: self.monitoring_tracer.update_local_events() - def _set_stopinfo(self, stopframe, returnframe, stoplineno=0, opcode=False): + def _set_stopinfo(self, stopframe, returnframe, stoplineno=0, opcode=False, + cmdframe=None, cmdlineno=None): """Set the attributes for stopping. If stoplineno is greater than or equal to 0, then stop at line @@ -539,6 +547,10 @@ def _set_stopinfo(self, stopframe, returnframe, stoplineno=0, opcode=False): # stoplineno >= 0 means: stop at line >= the stoplineno # stoplineno -1 means: don't stop at all self.stoplineno = stoplineno + # cmdframe/cmdlineno is the frame/line number when the user issued + # step/next commands. + self.cmdframe = cmdframe + self.cmdlineno = cmdlineno self._set_trace_opcodes(opcode) def _set_caller_tracefunc(self, current_frame): @@ -564,7 +576,9 @@ def set_until(self, frame, lineno=None): def set_step(self): """Stop after one line of code.""" - self._set_stopinfo(None, None) + # set_step() could be called from signal handler so enterframe might be None + self._set_stopinfo(None, None, cmdframe=self.enterframe, + cmdlineno=getattr(self.enterframe, 'f_lineno', None)) def set_stepinstr(self): """Stop before the next instruction.""" @@ -572,7 +586,7 @@ def set_stepinstr(self): def set_next(self, frame): """Stop on the next line in or below the given frame.""" - self._set_stopinfo(frame, None) + self._set_stopinfo(frame, None, cmdframe=frame, cmdlineno=frame.f_lineno) def set_return(self, frame): """Stop when returning from the given frame.""" diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 2ca689e0adf710..9d89008756a1d3 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -3232,6 +3232,37 @@ def test_pdb_issue_gh_127321(): """ +def test_pdb_issue_gh_136057(): + """See GH-136057 + "step" and "next" commands should be able to get over list comprehensions + >>> def test_function(): + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... lst = [i for i in range(10)] + ... for i in lst: pass + + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE + ... 'next', + ... 'next', + ... 'step', + ... 'continue', + ... ]): + ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) next + > (3)test_function() + -> lst = [i for i in range(10)] + (Pdb) next + > (4)test_function() + -> for i in lst: pass + (Pdb) step + --Return-- + > (4)test_function()->None + -> for i in lst: pass + (Pdb) continue + """ + + def test_pdb_issue_gh_80731(): """See GH-80731 diff --git a/Misc/NEWS.d/next/Library/2025-07-01-04-57-57.gh-issue-136057.4-t596.rst b/Misc/NEWS.d/next/Library/2025-07-01-04-57-57.gh-issue-136057.4-t596.rst new file mode 100644 index 00000000000000..e237a0e98cc486 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-01-04-57-57.gh-issue-136057.4-t596.rst @@ -0,0 +1 @@ +Fixed the bug in :mod:`pdb` and :mod:`bdb` where ``next`` and ``step`` can't go over the line if a loop exists in the line. From 7800b78067162fc9d7cb6926f703fe14dee1702a Mon Sep 17 00:00:00 2001 From: SubbaraoGarlapati <53627478+SubbaraoGarlapati@users.noreply.github.com> Date: Mon, 17 Nov 2025 06:23:12 -0500 Subject: [PATCH 068/638] fix memory order of `_Py_atomic_store_uint_release` (#141562) --- Include/cpython/pyatomic_std.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 69a8b9e615ea5f..7176f667a4082c 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -948,14 +948,6 @@ _Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value) memory_order_relaxed); } -static inline void -_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) -{ - _Py_USING_STD; - atomic_store_explicit((_Atomic(unsigned int)*)obj, value, - memory_order_relaxed); -} - static inline void _Py_atomic_store_long_relaxed(long *obj, long value) { @@ -1031,6 +1023,14 @@ _Py_atomic_store_int_release(int *obj, int value) memory_order_release); } +static inline void +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(unsigned int)*)obj, value, + memory_order_release); +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { From 31ea3f3c76b33e8e3cc098721266fe17f459e75d Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Mon, 17 Nov 2025 11:32:00 +0000 Subject: [PATCH 069/638] gh-141018: Update `.exe`, `.dll`, `.rtf` and `.jpg` mime types in `mimetypes` (#141023) --- Lib/mimetypes.py | 8 +++---- Lib/test/test_mimetypes.py | 24 ++++++++----------- ...-11-04-20-08-41.gh-issue-141018.d_oyOI.rst | 2 ++ 3 files changed, 15 insertions(+), 19 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-04-20-08-41.gh-issue-141018.d_oyOI.rst diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index 42477713c78418..07ac079186fbb7 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -489,8 +489,6 @@ def _default_mime_types(): '.cjs' : 'application/node', '.bin' : 'application/octet-stream', '.a' : 'application/octet-stream', - '.dll' : 'application/octet-stream', - '.exe' : 'application/octet-stream', '.o' : 'application/octet-stream', '.obj' : 'application/octet-stream', '.so' : 'application/octet-stream', @@ -501,12 +499,15 @@ def _default_mime_types(): '.p7c' : 'application/pkcs7-mime', '.ps' : 'application/postscript', '.eps' : 'application/postscript', + '.rtf' : 'application/rtf', '.texi' : 'application/texinfo', '.texinfo': 'application/texinfo', '.toml' : 'application/toml', '.trig' : 'application/trig', '.m3u' : 'application/vnd.apple.mpegurl', '.m3u8' : 'application/vnd.apple.mpegurl', + '.dll' : 'application/vnd.microsoft.portable-executable', + '.exe' : 'application/vnd.microsoft.portable-executable', '.xls' : 'application/vnd.ms-excel', '.xlb' : 'application/vnd.ms-excel', '.eot' : 'application/vnd.ms-fontobject', @@ -649,7 +650,6 @@ def _default_mime_types(): '.pl' : 'text/plain', '.srt' : 'text/plain', '.rtx' : 'text/richtext', - '.rtf' : 'text/rtf', '.tsv' : 'text/tab-separated-values', '.vtt' : 'text/vtt', '.py' : 'text/x-python', @@ -682,11 +682,9 @@ def _default_mime_types(): # Please sort these too common_types = _common_types_default = { - '.rtf' : 'application/rtf', '.apk' : 'application/vnd.android.package-archive', '.midi': 'audio/midi', '.mid' : 'audio/midi', - '.jpg' : 'image/jpg', '.pict': 'image/pict', '.pct' : 'image/pict', '.pic' : 'image/pict', diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py index 734144983591b4..0f29640bc1c494 100644 --- a/Lib/test/test_mimetypes.py +++ b/Lib/test/test_mimetypes.py @@ -112,13 +112,12 @@ def test_non_standard_types(self): eq = self.assertEqual # First try strict eq(self.db.guess_file_type('foo.xul', strict=True), (None, None)) - eq(self.db.guess_extension('image/jpg', strict=True), None) # And then non-strict eq(self.db.guess_file_type('foo.xul', strict=False), ('text/xul', None)) eq(self.db.guess_file_type('foo.XUL', strict=False), ('text/xul', None)) eq(self.db.guess_file_type('foo.invalid', strict=False), (None, None)) - eq(self.db.guess_extension('image/jpg', strict=False), '.jpg') - eq(self.db.guess_extension('image/JPG', strict=False), '.jpg') + eq(self.db.guess_extension('image/jpeg', strict=False), '.jpg') + eq(self.db.guess_extension('image/JPEG', strict=False), '.jpg') def test_filename_with_url_delimiters(self): # bpo-38449: URL delimiters cases should be handled also. @@ -179,8 +178,8 @@ def test_guess_all_types(self): self.assertTrue(set(all) >= {'.bat', '.c', '.h', '.ksh', '.pl', '.txt'}) self.assertEqual(len(set(all)), len(all)) # no duplicates # And now non-strict - all = self.db.guess_all_extensions('image/jpg', strict=False) - self.assertEqual(all, ['.jpg']) + all = self.db.guess_all_extensions('image/jpeg', strict=False) + self.assertEqual(all, ['.jpg', '.jpe', '.jpeg']) # And now for no hits all = self.db.guess_all_extensions('image/jpg', strict=True) self.assertEqual(all, []) @@ -231,6 +230,7 @@ def check_extensions(): ("application/ogg", ".ogx"), ("application/pdf", ".pdf"), ("application/postscript", ".ps"), + ("application/rtf", ".rtf"), ("application/texinfo", ".texi"), ("application/toml", ".toml"), ("application/vnd.apple.mpegurl", ".m3u"), @@ -281,7 +281,6 @@ def check_extensions(): ("model/stl", ".stl"), ("text/html", ".html"), ("text/plain", ".txt"), - ("text/rtf", ".rtf"), ("text/x-rst", ".rst"), ("video/matroska", ".mkv"), ("video/matroska-3d", ".mk3d"), @@ -372,9 +371,7 @@ def test_keywords_args_api(self): self.assertEqual(self.db.guess_type( url="scheme:foo.html", strict=True), ("text/html", None)) self.assertEqual(self.db.guess_all_extensions( - type='image/jpg', strict=True), []) - self.assertEqual(self.db.guess_extension( - type='image/jpg', strict=False), '.jpg') + type='image/jpeg', strict=True), ['.jpg', '.jpe', '.jpeg']) def test_added_types_are_used(self): mimetypes.add_type('testing/default-type', '') @@ -452,15 +449,15 @@ def test_parse_args(self): args, help_text = mimetypes._parse_args("--invalid") self.assertTrue(help_text.startswith("usage: ")) - args, _ = mimetypes._parse_args(shlex.split("-l -e image/jpg")) + args, _ = mimetypes._parse_args(shlex.split("-l -e image/jpeg")) self.assertTrue(args.extension) self.assertTrue(args.lenient) - self.assertEqual(args.type, ["image/jpg"]) + self.assertEqual(args.type, ["image/jpeg"]) - args, _ = mimetypes._parse_args(shlex.split("-e image/jpg")) + args, _ = mimetypes._parse_args(shlex.split("-e image/jpeg")) self.assertTrue(args.extension) self.assertFalse(args.lenient) - self.assertEqual(args.type, ["image/jpg"]) + self.assertEqual(args.type, ["image/jpeg"]) args, _ = mimetypes._parse_args(shlex.split("-l foo.webp")) self.assertFalse(args.extension) @@ -491,7 +488,6 @@ def test_multiple_inputs_error(self): def test_invocation(self): for command, expected in [ - ("-l -e image/jpg", ".jpg"), ("-e image/jpeg", ".jpg"), ("-l foo.webp", "type: image/webp encoding: None"), ]: diff --git a/Misc/NEWS.d/next/Library/2025-11-04-20-08-41.gh-issue-141018.d_oyOI.rst b/Misc/NEWS.d/next/Library/2025-11-04-20-08-41.gh-issue-141018.d_oyOI.rst new file mode 100644 index 00000000000000..e776515a9fb267 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-04-20-08-41.gh-issue-141018.d_oyOI.rst @@ -0,0 +1,2 @@ +:mod:`mimetypes`: Update ``.exe``, ``.dll``, ``.rtf`` and (when +``strict=False``) ``.jpg`` to their correct IANA mime type. From df8091d516f874bd8222569794229ea77fb3a0a3 Mon Sep 17 00:00:00 2001 From: Tamzin Hadasa Kelly Date: Mon, 17 Nov 2025 18:35:01 +0700 Subject: [PATCH 070/638] gh-141650: Fix typo in `xml.sax.saxutils.unescape` documentation (#141652) --- Doc/library/xml.sax.utils.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/xml.sax.utils.rst b/Doc/library/xml.sax.utils.rst index 5ee11d58c3dd26..7731f03d875efc 100644 --- a/Doc/library/xml.sax.utils.rst +++ b/Doc/library/xml.sax.utils.rst @@ -37,7 +37,7 @@ or as base classes. You can unescape other strings of data by passing a dictionary as the optional *entities* parameter. The keys and values must all be strings; each key will be - replaced with its corresponding value. ``'&'``, ``'<'``, and ``'>'`` + replaced with its corresponding value. ``'&'``, ``'<'``, and ``'>'`` are always unescaped, even if *entities* is provided. From d527d3bf8beb9cd26c179f2c0111d635cdaa9cd3 Mon Sep 17 00:00:00 2001 From: dereckduran <67027239+dereckduran@users.noreply.github.com> Date: Mon, 17 Nov 2025 03:44:44 -0800 Subject: [PATCH 071/638] gh-62480: De-personalize "Coping with mutable arguments" section in `unittest.mock` examples (#141323) --- Doc/library/unittest.mock-examples.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/unittest.mock-examples.rst b/Doc/library/unittest.mock-examples.rst index 6af4298d44f532..61c75b5a03b103 100644 --- a/Doc/library/unittest.mock-examples.rst +++ b/Doc/library/unittest.mock-examples.rst @@ -863,9 +863,9 @@ Here's one solution that uses the :attr:`~Mock.side_effect` functionality. If you provide a ``side_effect`` function for a mock then ``side_effect`` will be called with the same args as the mock. This gives us an opportunity to copy the arguments and store them for later assertions. In this -example I'm using *another* mock to store the arguments so that I can use the +example we're using *another* mock to store the arguments so that we can use the mock methods for doing the assertion. Again a helper function sets this up for -me. :: +us. :: >>> from copy import deepcopy >>> from unittest.mock import Mock, patch, DEFAULT From 20b64bdf23b88e44f72bc49f8bc783ae8ca21511 Mon Sep 17 00:00:00 2001 From: Thomas Ballard Date: Mon, 17 Nov 2025 06:47:28 -0500 Subject: [PATCH 072/638] Docs: Fix typo in socketserver documentation (#140956) --- Doc/library/socketserver.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/socketserver.rst b/Doc/library/socketserver.rst index 7bc2f7afbbb0b1..491b8769f44fe2 100644 --- a/Doc/library/socketserver.rst +++ b/Doc/library/socketserver.rst @@ -546,7 +546,7 @@ The difference is that the ``readline()`` call in the second handler will call first handler had to use a ``recv()`` loop to accumulate data until a newline itself. If it had just used a single ``recv()`` without the loop it would just have returned what has been received so far from the client. -TCP is stream based: data arrives in the order it was sent, but there no +TCP is stream based: data arrives in the order it was sent, but there is no correlation between client ``send()`` or ``sendall()`` calls and the number of ``recv()`` calls on the server required to receive it. From 994ab5c922b179ab1884f05b3440c24db9e9733d Mon Sep 17 00:00:00 2001 From: yihong Date: Mon, 17 Nov 2025 20:43:14 +0800 Subject: [PATCH 073/638] gh-140729: Add __mp_main__ as a duplicate for __main__ for pickle to work (#140735) --- Lib/profiling/sampling/_sync_coordinator.py | 11 +++- .../test_profiling/test_sampling_profiler.py | 52 ++++++++++++++++++- ...-10-29-11-31-59.gh-issue-140729.t9JsNt.rst | 2 + 3 files changed, 62 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-11-31-59.gh-issue-140729.t9JsNt.rst diff --git a/Lib/profiling/sampling/_sync_coordinator.py b/Lib/profiling/sampling/_sync_coordinator.py index adb040e89cc7b1..be63dbe3e904ce 100644 --- a/Lib/profiling/sampling/_sync_coordinator.py +++ b/Lib/profiling/sampling/_sync_coordinator.py @@ -10,6 +10,7 @@ import socket import runpy import time +import types from typing import List, NoReturn @@ -175,15 +176,21 @@ def _execute_script(script_path: str, script_args: List[str], cwd: str) -> None: try: with open(script_path, 'rb') as f: source_code = f.read() + except FileNotFoundError as e: raise TargetError(f"Script file not found: {script_path}") from e except PermissionError as e: raise TargetError(f"Permission denied reading script: {script_path}") from e try: - # Compile and execute the script + main_module = types.ModuleType("__main__") + main_module.__file__ = script_path + main_module.__builtins__ = __builtins__ + # gh-140729: Create a __mp_main__ module to allow pickling + sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module + code = compile(source_code, script_path, 'exec', module='__main__') - exec(code, {'__name__': '__main__', '__file__': script_path}) + exec(code, main_module.__dict__) except SyntaxError as e: raise TargetError(f"Syntax error in script {script_path}: {e}") from e except SystemExit: diff --git a/Lib/test/test_profiling/test_sampling_profiler.py b/Lib/test/test_profiling/test_sampling_profiler.py index 5b924cb24531b6..0ba6799a1ce5ba 100644 --- a/Lib/test/test_profiling/test_sampling_profiler.py +++ b/Lib/test/test_profiling/test_sampling_profiler.py @@ -22,7 +22,13 @@ from profiling.sampling.gecko_collector import GeckoCollector from test.support.os_helper import unlink -from test.support import force_not_colorized_test_class, SHORT_TIMEOUT +from test.support import ( + force_not_colorized_test_class, + SHORT_TIMEOUT, + script_helper, + os_helper, + SuppressCrashReport, +) from test.support.socket_helper import find_unused_port from test.support import requires_subprocess, is_emscripten from test.support import captured_stdout, captured_stderr @@ -3009,5 +3015,49 @@ def test_parse_mode_function(self): profiling.sampling.sample._parse_mode("invalid") +@requires_subprocess() +@skip_if_not_supported +class TestProcessPoolExecutorSupport(unittest.TestCase): + """ + Test that ProcessPoolExecutor works correctly with profiling.sampling. + """ + + def test_process_pool_executor_pickle(self): + # gh-140729: test use ProcessPoolExecutor.map() can sampling + test_script = ''' +import concurrent.futures + +def worker(x): + return x * 2 + +if __name__ == "__main__": + with concurrent.futures.ProcessPoolExecutor() as executor: + results = list(executor.map(worker, [1, 2, 3])) + print(f"Results: {results}") +''' + with os_helper.temp_dir() as temp_dir: + script = script_helper.make_script( + temp_dir, 'test_process_pool_executor_pickle', test_script + ) + with SuppressCrashReport(): + with script_helper.spawn_python( + "-m", "profiling.sampling.sample", + "-d", "5", + "-i", "100000", + script, + stderr=subprocess.PIPE, + text=True + ) as proc: + proc.wait(timeout=SHORT_TIMEOUT) + stdout = proc.stdout.read() + stderr = proc.stderr.read() + + if "PermissionError" in stderr: + self.skipTest("Insufficient permissions for remote profiling") + + self.assertIn("Results: [2, 4, 6]", stdout) + self.assertNotIn("Can't pickle", stderr) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-11-31-59.gh-issue-140729.t9JsNt.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-11-31-59.gh-issue-140729.t9JsNt.rst new file mode 100644 index 00000000000000..6725547667fb3c --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-11-31-59.gh-issue-140729.t9JsNt.rst @@ -0,0 +1,2 @@ +Fix pickling error in the sampling profiler when using ``concurrent.futures.ProcessPoolExecutor`` +script can not be properly pickled and executed in worker processes. From 89a914c58db1661cb9da4f3b9e52c20bb4b02287 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Mon, 17 Nov 2025 12:46:26 +0000 Subject: [PATCH 074/638] gh-135953: Add GIL contention markers to sampling profiler Gecko format (#139485) This commit enhances the Gecko format reporter in the sampling profiler to include markers for GIL acquisition events. --- Include/cpython/pystate.h | 3 + Include/internal/pycore_debug_offsets.h | 4 + Lib/profiling/sampling/collector.py | 31 +-- Lib/profiling/sampling/gecko_collector.py | 237 ++++++++++++++++-- Lib/profiling/sampling/sample.py | 37 ++- Lib/test/test_external_inspection.py | 154 +++++++++++- .../test_profiling/test_sampling_profiler.py | 116 ++++++++- Modules/_remote_debugging_module.c | 123 +++++++-- Python/ceval_gil.c | 4 + 9 files changed, 627 insertions(+), 82 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index c53abe43ebe65c..1e1e46ea4c0bcd 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -113,6 +113,9 @@ struct _ts { /* Currently holds the GIL. Must be its own field to avoid data races */ int holds_gil; + /* Currently requesting the GIL */ + int gil_requested; + int _whence; /* Thread state (_Py_THREAD_ATTACHED, _Py_THREAD_DETACHED, _Py_THREAD_SUSPENDED). diff --git a/Include/internal/pycore_debug_offsets.h b/Include/internal/pycore_debug_offsets.h index 8e7cd16acffa48..f6d50bf5df7a9e 100644 --- a/Include/internal/pycore_debug_offsets.h +++ b/Include/internal/pycore_debug_offsets.h @@ -106,6 +106,8 @@ typedef struct _Py_DebugOffsets { uint64_t native_thread_id; uint64_t datastack_chunk; uint64_t status; + uint64_t holds_gil; + uint64_t gil_requested; } thread_state; // InterpreterFrame offset; @@ -273,6 +275,8 @@ typedef struct _Py_DebugOffsets { .native_thread_id = offsetof(PyThreadState, native_thread_id), \ .datastack_chunk = offsetof(PyThreadState, datastack_chunk), \ .status = offsetof(PyThreadState, _status), \ + .holds_gil = offsetof(PyThreadState, holds_gil), \ + .gil_requested = offsetof(PyThreadState, gil_requested), \ }, \ .interpreter_frame = { \ .size = sizeof(_PyInterpreterFrame), \ diff --git a/Lib/profiling/sampling/collector.py b/Lib/profiling/sampling/collector.py index b7a033ac0a6637..3c2325ef77268c 100644 --- a/Lib/profiling/sampling/collector.py +++ b/Lib/profiling/sampling/collector.py @@ -1,17 +1,14 @@ from abc import ABC, abstractmethod -# Enums are slow -THREAD_STATE_RUNNING = 0 -THREAD_STATE_IDLE = 1 -THREAD_STATE_GIL_WAIT = 2 -THREAD_STATE_UNKNOWN = 3 - -STATUS = { - THREAD_STATE_RUNNING: "running", - THREAD_STATE_IDLE: "idle", - THREAD_STATE_GIL_WAIT: "gil_wait", - THREAD_STATE_UNKNOWN: "unknown", -} +# Thread status flags +try: + from _remote_debugging import THREAD_STATUS_HAS_GIL, THREAD_STATUS_ON_CPU, THREAD_STATUS_UNKNOWN, THREAD_STATUS_GIL_REQUESTED +except ImportError: + # Fallback for tests or when module is not available + THREAD_STATUS_HAS_GIL = (1 << 0) + THREAD_STATUS_ON_CPU = (1 << 1) + THREAD_STATUS_UNKNOWN = (1 << 2) + THREAD_STATUS_GIL_REQUESTED = (1 << 3) class Collector(ABC): @abstractmethod @@ -26,8 +23,14 @@ def _iter_all_frames(self, stack_frames, skip_idle=False): """Iterate over all frame stacks from all interpreters and threads.""" for interpreter_info in stack_frames: for thread_info in interpreter_info.threads: - if skip_idle and thread_info.status != THREAD_STATE_RUNNING: - continue + # skip_idle now means: skip if thread is not actively running + # A thread is "active" if it has the GIL OR is on CPU + if skip_idle: + status_flags = thread_info.status + has_gil = bool(status_flags & THREAD_STATUS_HAS_GIL) + on_cpu = bool(status_flags & THREAD_STATUS_ON_CPU) + if not (has_gil or on_cpu): + continue frames = thread_info.frame_info if frames: yield frames, thread_info.thread_id diff --git a/Lib/profiling/sampling/gecko_collector.py b/Lib/profiling/sampling/gecko_collector.py index 548acbf24b7fd2..6c6700f113083e 100644 --- a/Lib/profiling/sampling/gecko_collector.py +++ b/Lib/profiling/sampling/gecko_collector.py @@ -1,9 +1,20 @@ +import itertools import json import os import platform +import sys +import threading import time -from .collector import Collector, THREAD_STATE_RUNNING +from .collector import Collector +try: + from _remote_debugging import THREAD_STATUS_HAS_GIL, THREAD_STATUS_ON_CPU, THREAD_STATUS_UNKNOWN, THREAD_STATUS_GIL_REQUESTED +except ImportError: + # Fallback if module not available (shouldn't happen in normal use) + THREAD_STATUS_HAS_GIL = (1 << 0) + THREAD_STATUS_ON_CPU = (1 << 1) + THREAD_STATUS_UNKNOWN = (1 << 2) + THREAD_STATUS_GIL_REQUESTED = (1 << 3) # Categories matching Firefox Profiler expectations @@ -11,14 +22,20 @@ {"name": "Other", "color": "grey", "subcategories": ["Other"]}, {"name": "Python", "color": "yellow", "subcategories": ["Other"]}, {"name": "Native", "color": "blue", "subcategories": ["Other"]}, - {"name": "Idle", "color": "transparent", "subcategories": ["Other"]}, + {"name": "GC", "color": "orange", "subcategories": ["Other"]}, + {"name": "GIL", "color": "green", "subcategories": ["Other"]}, + {"name": "CPU", "color": "purple", "subcategories": ["Other"]}, + {"name": "Code Type", "color": "red", "subcategories": ["Other"]}, ] # Category indices CATEGORY_OTHER = 0 CATEGORY_PYTHON = 1 CATEGORY_NATIVE = 2 -CATEGORY_IDLE = 3 +CATEGORY_GC = 3 +CATEGORY_GIL = 4 +CATEGORY_CPU = 5 +CATEGORY_CODE_TYPE = 6 # Subcategory indices DEFAULT_SUBCATEGORY = 0 @@ -58,6 +75,56 @@ def __init__(self, *, skip_idle=False): self.last_sample_time = 0 self.interval = 1.0 # Will be calculated from actual sampling + # State tracking for interval markers (tid -> start_time) + self.has_gil_start = {} # Thread has the GIL + self.no_gil_start = {} # Thread doesn't have the GIL + self.on_cpu_start = {} # Thread is running on CPU + self.off_cpu_start = {} # Thread is off CPU + self.python_code_start = {} # Thread running Python code (has GIL) + self.native_code_start = {} # Thread running native code (on CPU without GIL) + self.gil_wait_start = {} # Thread waiting for GIL + + # GC event tracking: track GC start time per thread + self.gc_start_per_thread = {} # tid -> start_time + + # Track which threads have been initialized for state tracking + self.initialized_threads = set() + + def _track_state_transition(self, tid, condition, active_dict, inactive_dict, + active_name, inactive_name, category, current_time): + """Track binary state transitions and emit markers. + + Args: + tid: Thread ID + condition: Whether the active state is true + active_dict: Dict tracking start time of active state + inactive_dict: Dict tracking start time of inactive state + active_name: Name for active state marker + inactive_name: Name for inactive state marker + category: Gecko category for the markers + current_time: Current timestamp + """ + # On first observation of a thread, just record the current state + # without creating a marker (we don't know what the previous state was) + if tid not in self.initialized_threads: + if condition: + active_dict[tid] = current_time + else: + inactive_dict[tid] = current_time + return + + # For already-initialized threads, track transitions + if condition: + active_dict.setdefault(tid, current_time) + if tid in inactive_dict: + self._add_marker(tid, inactive_name, inactive_dict.pop(tid), + current_time, category) + else: + inactive_dict.setdefault(tid, current_time) + if tid in active_dict: + self._add_marker(tid, active_name, active_dict.pop(tid), + current_time, category) + def collect(self, stack_frames): """Collect a sample from stack frames.""" current_time = (time.time() * 1000) - self.start_time @@ -69,19 +136,12 @@ def collect(self, stack_frames): ) / self.sample_count self.last_sample_time = current_time + # Process threads and track GC per thread for interpreter_info in stack_frames: for thread_info in interpreter_info.threads: - if ( - self.skip_idle - and thread_info.status != THREAD_STATE_RUNNING - ): - continue - frames = thread_info.frame_info - if not frames: - continue - tid = thread_info.thread_id + gc_collecting = thread_info.gc_collecting # Initialize thread if needed if tid not in self.threads: @@ -89,6 +149,80 @@ def collect(self, stack_frames): thread_data = self.threads[tid] + # Decode status flags + status_flags = thread_info.status + has_gil = bool(status_flags & THREAD_STATUS_HAS_GIL) + on_cpu = bool(status_flags & THREAD_STATUS_ON_CPU) + gil_requested = bool(status_flags & THREAD_STATUS_GIL_REQUESTED) + + # Track GIL possession (Has GIL / No GIL) + self._track_state_transition( + tid, has_gil, self.has_gil_start, self.no_gil_start, + "Has GIL", "No GIL", CATEGORY_GIL, current_time + ) + + # Track CPU state (On CPU / Off CPU) + self._track_state_transition( + tid, on_cpu, self.on_cpu_start, self.off_cpu_start, + "On CPU", "Off CPU", CATEGORY_CPU, current_time + ) + + # Track code type (Python Code / Native Code) + # This is tri-state: Python (has_gil), Native (on_cpu without gil), or Neither + if has_gil: + self._track_state_transition( + tid, True, self.python_code_start, self.native_code_start, + "Python Code", "Native Code", CATEGORY_CODE_TYPE, current_time + ) + elif on_cpu: + self._track_state_transition( + tid, True, self.native_code_start, self.python_code_start, + "Native Code", "Python Code", CATEGORY_CODE_TYPE, current_time + ) + else: + # Thread is idle (neither has GIL nor on CPU) - close any open code markers + # This handles the third state that _track_state_transition doesn't cover + if tid in self.initialized_threads: + if tid in self.python_code_start: + self._add_marker(tid, "Python Code", self.python_code_start.pop(tid), + current_time, CATEGORY_CODE_TYPE) + if tid in self.native_code_start: + self._add_marker(tid, "Native Code", self.native_code_start.pop(tid), + current_time, CATEGORY_CODE_TYPE) + + # Track "Waiting for GIL" intervals (one-sided tracking) + if gil_requested: + self.gil_wait_start.setdefault(tid, current_time) + elif tid in self.gil_wait_start: + self._add_marker(tid, "Waiting for GIL", self.gil_wait_start.pop(tid), + current_time, CATEGORY_GIL) + + # Track GC events - attribute to all threads that hold the GIL during GC + # (GC is interpreter-wide but runs on whichever thread(s) have the GIL) + # If GIL switches during GC, multiple threads will get GC markers + if gc_collecting and has_gil: + # Start GC marker if not already started for this thread + if tid not in self.gc_start_per_thread: + self.gc_start_per_thread[tid] = current_time + elif tid in self.gc_start_per_thread: + # End GC marker if it was running for this thread + # (either GC finished or thread lost GIL) + self._add_marker(tid, "GC Collecting", self.gc_start_per_thread.pop(tid), + current_time, CATEGORY_GC) + + # Mark thread as initialized after processing all state transitions + self.initialized_threads.add(tid) + + # Categorize: idle if neither has GIL nor on CPU + is_idle = not has_gil and not on_cpu + + # Skip idle threads if skip_idle is enabled + if self.skip_idle and is_idle: + continue + + if not frames: + continue + # Process the stack stack_index = self._process_stack(thread_data, frames) @@ -102,7 +236,6 @@ def collect(self, stack_frames): def _create_thread(self, tid): """Create a new thread structure with processed profile format.""" - import threading # Determine if this is the main thread try: @@ -181,7 +314,7 @@ def _create_thread(self, tid): "functionSize": [], "length": 0, }, - # Markers - processed format + # Markers - processed format (arrays) "markers": { "data": [], "name": [], @@ -215,6 +348,27 @@ def _intern_string(self, s): self.global_string_map[s] = idx return idx + def _add_marker(self, tid, name, start_time, end_time, category): + """Add an interval marker for a specific thread.""" + if tid not in self.threads: + return + + thread_data = self.threads[tid] + duration = end_time - start_time + + name_idx = self._intern_string(name) + markers = thread_data["markers"] + markers["name"].append(name_idx) + markers["startTime"].append(start_time) + markers["endTime"].append(end_time) + markers["phase"].append(1) # 1 = interval marker + markers["category"].append(category) + markers["data"].append({ + "type": name.replace(" ", ""), + "duration": duration, + "tid": tid + }) + def _process_stack(self, thread_data, frames): """Process a stack and return the stack index.""" if not frames: @@ -383,15 +537,63 @@ def _get_or_create_frame(self, thread_data, func_idx, lineno): frame_cache[frame_key] = frame_idx return frame_idx + def _finalize_markers(self): + """Close any open markers at the end of profiling.""" + end_time = self.last_sample_time + + # Close all open markers for each thread using a generic approach + marker_states = [ + (self.has_gil_start, "Has GIL", CATEGORY_GIL), + (self.no_gil_start, "No GIL", CATEGORY_GIL), + (self.on_cpu_start, "On CPU", CATEGORY_CPU), + (self.off_cpu_start, "Off CPU", CATEGORY_CPU), + (self.python_code_start, "Python Code", CATEGORY_CODE_TYPE), + (self.native_code_start, "Native Code", CATEGORY_CODE_TYPE), + (self.gil_wait_start, "Waiting for GIL", CATEGORY_GIL), + (self.gc_start_per_thread, "GC Collecting", CATEGORY_GC), + ] + + for state_dict, marker_name, category in marker_states: + for tid in list(state_dict.keys()): + self._add_marker(tid, marker_name, state_dict[tid], end_time, category) + del state_dict[tid] + def export(self, filename): """Export the profile to a Gecko JSON file.""" + if self.sample_count > 0 and self.last_sample_time > 0: self.interval = self.last_sample_time / self.sample_count - profile = self._build_profile() + # Spinner for progress indication + spinner = itertools.cycle(['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏']) + stop_spinner = threading.Event() + + def spin(): + message = 'Building Gecko profile...' + while not stop_spinner.is_set(): + sys.stderr.write(f'\r{next(spinner)} {message}') + sys.stderr.flush() + time.sleep(0.1) + # Clear the spinner line + sys.stderr.write('\r' + ' ' * (len(message) + 3) + '\r') + sys.stderr.flush() + + spinner_thread = threading.Thread(target=spin, daemon=True) + spinner_thread.start() + + try: + # Finalize any open markers before building profile + self._finalize_markers() + + profile = self._build_profile() - with open(filename, "w") as f: - json.dump(profile, f, separators=(",", ":")) + with open(filename, "w") as f: + json.dump(profile, f, separators=(",", ":")) + finally: + stop_spinner.set() + spinner_thread.join(timeout=1.0) + # Small delay to ensure the clear happens + time.sleep(0.01) print(f"Gecko profile written to {filename}") print( @@ -416,6 +618,7 @@ def _build_profile(self): frame_table["length"] = len(frame_table["func"]) func_table["length"] = len(func_table["name"]) resource_table["length"] = len(resource_table["name"]) + thread_data["markers"]["length"] = len(thread_data["markers"]["name"]) # Clean up internal caches del thread_data["_stackCache"] diff --git a/Lib/profiling/sampling/sample.py b/Lib/profiling/sampling/sample.py index 7a0f739a5428c6..5ca68911d8a482 100644 --- a/Lib/profiling/sampling/sample.py +++ b/Lib/profiling/sampling/sample.py @@ -21,6 +21,7 @@ PROFILING_MODE_WALL = 0 PROFILING_MODE_CPU = 1 PROFILING_MODE_GIL = 2 +PROFILING_MODE_ALL = 3 # Combines GIL + CPU checks def _parse_mode(mode_string): @@ -136,18 +137,20 @@ def _run_with_sync(original_cmd): class SampleProfiler: - def __init__(self, pid, sample_interval_usec, all_threads, *, mode=PROFILING_MODE_WALL): + def __init__(self, pid, sample_interval_usec, all_threads, *, mode=PROFILING_MODE_WALL, skip_non_matching_threads=True): self.pid = pid self.sample_interval_usec = sample_interval_usec self.all_threads = all_threads if _FREE_THREADED_BUILD: self.unwinder = _remote_debugging.RemoteUnwinder( - self.pid, all_threads=self.all_threads, mode=mode + self.pid, all_threads=self.all_threads, mode=mode, + skip_non_matching_threads=skip_non_matching_threads ) else: only_active_threads = bool(self.all_threads) self.unwinder = _remote_debugging.RemoteUnwinder( - self.pid, only_active_thread=only_active_threads, mode=mode + self.pid, only_active_thread=only_active_threads, mode=mode, + skip_non_matching_threads=skip_non_matching_threads ) # Track sample intervals and total sample count self.sample_intervals = deque(maxlen=100) @@ -614,14 +617,21 @@ def sample( realtime_stats=False, mode=PROFILING_MODE_WALL, ): + # PROFILING_MODE_ALL implies no skipping at all + if mode == PROFILING_MODE_ALL: + skip_non_matching_threads = False + skip_idle = False + else: + # Determine skip settings based on output format and mode + skip_non_matching_threads = output_format != "gecko" + skip_idle = mode != PROFILING_MODE_WALL + profiler = SampleProfiler( - pid, sample_interval_usec, all_threads=all_threads, mode=mode + pid, sample_interval_usec, all_threads=all_threads, mode=mode, + skip_non_matching_threads=skip_non_matching_threads ) profiler.realtime_stats = realtime_stats - # Determine skip_idle for collector compatibility - skip_idle = mode != PROFILING_MODE_WALL - collector = None match output_format: case "pstats": @@ -633,7 +643,8 @@ def sample( collector = FlamegraphCollector(skip_idle=skip_idle) filename = filename or f"flamegraph.{pid}.html" case "gecko": - collector = GeckoCollector(skip_idle=skip_idle) + # Gecko format never skips idle threads to show full thread states + collector = GeckoCollector(skip_idle=False) filename = filename or f"gecko.{pid}.json" case _: raise ValueError(f"Invalid output format: {output_format}") @@ -882,6 +893,10 @@ def main(): if args.format in ("collapsed", "gecko"): _validate_collapsed_format_args(args, parser) + # Validate that --mode is not used with --gecko + if args.format == "gecko" and args.mode != "wall": + parser.error("--mode option is incompatible with --gecko format. Gecko format automatically uses ALL mode (GIL + CPU analysis).") + sort_value = args.sort if args.sort is not None else 2 if args.module is not None and not args.module: @@ -900,7 +915,11 @@ def main(): elif target_count > 1: parser.error("only one target type can be specified: -p/--pid, -m/--module, or script") - mode = _parse_mode(args.mode) + # Use PROFILING_MODE_ALL for gecko format, otherwise parse user's choice + if args.format == "gecko": + mode = PROFILING_MODE_ALL + else: + mode = _parse_mode(args.mode) if args.pid: sample( diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 01720457e61f5c..60e5000cd72a32 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -23,6 +23,12 @@ PROFILING_MODE_WALL = 0 PROFILING_MODE_CPU = 1 PROFILING_MODE_GIL = 2 +PROFILING_MODE_ALL = 3 + +# Thread status flags +THREAD_STATUS_HAS_GIL = (1 << 0) +THREAD_STATUS_ON_CPU = (1 << 1) +THREAD_STATUS_UNKNOWN = (1 << 2) try: from concurrent import interpreters @@ -1763,11 +1769,14 @@ def busy(): for thread_info in interpreter_info.threads: statuses[thread_info.thread_id] = thread_info.status - # Check if sleeper thread is idle and busy thread is running + # Check if sleeper thread is off CPU and busy thread is on CPU + # In the new flags system: + # - sleeper should NOT have ON_CPU flag (off CPU) + # - busy should have ON_CPU flag if (sleeper_tid in statuses and busy_tid in statuses and - statuses[sleeper_tid] == 1 and - statuses[busy_tid] == 0): + not (statuses[sleeper_tid] & THREAD_STATUS_ON_CPU) and + (statuses[busy_tid] & THREAD_STATUS_ON_CPU)): break time.sleep(0.5) # Give a bit of time to let threads settle except PermissionError: @@ -1779,8 +1788,8 @@ def busy(): self.assertIsNotNone(busy_tid, "Busy thread id not received") self.assertIn(sleeper_tid, statuses, "Sleeper tid not found in sampled threads") self.assertIn(busy_tid, statuses, "Busy tid not found in sampled threads") - self.assertEqual(statuses[sleeper_tid], 1, "Sleeper thread should be idle (1)") - self.assertEqual(statuses[busy_tid], 0, "Busy thread should be running (0)") + self.assertFalse(statuses[sleeper_tid] & THREAD_STATUS_ON_CPU, "Sleeper thread should be off CPU") + self.assertTrue(statuses[busy_tid] & THREAD_STATUS_ON_CPU, "Busy thread should be on CPU") finally: if client_socket is not None: @@ -1875,11 +1884,14 @@ def busy(): for thread_info in interpreter_info.threads: statuses[thread_info.thread_id] = thread_info.status - # Check if sleeper thread is idle (status 2 for GIL mode) and busy thread is running + # Check if sleeper thread doesn't have GIL and busy thread has GIL + # In the new flags system: + # - sleeper should NOT have HAS_GIL flag (waiting for GIL) + # - busy should have HAS_GIL flag if (sleeper_tid in statuses and busy_tid in statuses and - statuses[sleeper_tid] == 2 and - statuses[busy_tid] == 0): + not (statuses[sleeper_tid] & THREAD_STATUS_HAS_GIL) and + (statuses[busy_tid] & THREAD_STATUS_HAS_GIL)): break time.sleep(0.5) # Give a bit of time to let threads settle except PermissionError: @@ -1891,8 +1903,8 @@ def busy(): self.assertIsNotNone(busy_tid, "Busy thread id not received") self.assertIn(sleeper_tid, statuses, "Sleeper tid not found in sampled threads") self.assertIn(busy_tid, statuses, "Busy tid not found in sampled threads") - self.assertEqual(statuses[sleeper_tid], 2, "Sleeper thread should be idle (1)") - self.assertEqual(statuses[busy_tid], 0, "Busy thread should be running (0)") + self.assertFalse(statuses[sleeper_tid] & THREAD_STATUS_HAS_GIL, "Sleeper thread should not have GIL") + self.assertTrue(statuses[busy_tid] & THREAD_STATUS_HAS_GIL, "Busy thread should have GIL") finally: if client_socket is not None: @@ -1900,6 +1912,128 @@ def busy(): p.terminate() p.wait(timeout=SHORT_TIMEOUT) + @unittest.skipIf( + sys.platform not in ("linux", "darwin", "win32"), + "Test only runs on supported platforms (Linux, macOS, or Windows)", + ) + @unittest.skipIf(sys.platform == "android", "Android raises Linux-specific exception") + def test_thread_status_all_mode_detection(self): + port = find_unused_port() + script = textwrap.dedent( + f"""\ + import socket + import threading + import time + import sys + + def sleeper_thread(): + conn = socket.create_connection(("localhost", {port})) + conn.sendall(b"sleeper:" + str(threading.get_native_id()).encode()) + while True: + time.sleep(1) + + def busy_thread(): + conn = socket.create_connection(("localhost", {port})) + conn.sendall(b"busy:" + str(threading.get_native_id()).encode()) + while True: + sum(range(100000)) + + t1 = threading.Thread(target=sleeper_thread) + t2 = threading.Thread(target=busy_thread) + t1.start() + t2.start() + t1.join() + t2.join() + """ + ) + + with os_helper.temp_dir() as tmp_dir: + script_file = make_script(tmp_dir, "script", script) + server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server_socket.bind(("localhost", port)) + server_socket.listen(2) + server_socket.settimeout(SHORT_TIMEOUT) + + p = subprocess.Popen( + [sys.executable, script_file], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + client_sockets = [] + try: + sleeper_tid = None + busy_tid = None + + # Receive thread IDs from the child process + for _ in range(2): + client_socket, _ = server_socket.accept() + client_sockets.append(client_socket) + line = client_socket.recv(1024) + if line: + if line.startswith(b"sleeper:"): + try: + sleeper_tid = int(line.split(b":")[-1]) + except Exception: + pass + elif line.startswith(b"busy:"): + try: + busy_tid = int(line.split(b":")[-1]) + except Exception: + pass + + server_socket.close() + + attempts = 10 + statuses = {} + try: + unwinder = RemoteUnwinder(p.pid, all_threads=True, mode=PROFILING_MODE_ALL, + skip_non_matching_threads=False) + for _ in range(attempts): + traces = unwinder.get_stack_trace() + # Find threads and their statuses + statuses = {} + for interpreter_info in traces: + for thread_info in interpreter_info.threads: + statuses[thread_info.thread_id] = thread_info.status + + # Check ALL mode provides both GIL and CPU info + # - sleeper should NOT have ON_CPU and NOT have HAS_GIL + # - busy should have ON_CPU and have HAS_GIL + if (sleeper_tid in statuses and + busy_tid in statuses and + not (statuses[sleeper_tid] & THREAD_STATUS_ON_CPU) and + not (statuses[sleeper_tid] & THREAD_STATUS_HAS_GIL) and + (statuses[busy_tid] & THREAD_STATUS_ON_CPU) and + (statuses[busy_tid] & THREAD_STATUS_HAS_GIL)): + break + time.sleep(0.5) + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" + ) + + self.assertIsNotNone(sleeper_tid, "Sleeper thread id not received") + self.assertIsNotNone(busy_tid, "Busy thread id not received") + self.assertIn(sleeper_tid, statuses, "Sleeper tid not found in sampled threads") + self.assertIn(busy_tid, statuses, "Busy tid not found in sampled threads") + + # Sleeper thread: off CPU, no GIL + self.assertFalse(statuses[sleeper_tid] & THREAD_STATUS_ON_CPU, "Sleeper should be off CPU") + self.assertFalse(statuses[sleeper_tid] & THREAD_STATUS_HAS_GIL, "Sleeper should not have GIL") + + # Busy thread: on CPU, has GIL + self.assertTrue(statuses[busy_tid] & THREAD_STATUS_ON_CPU, "Busy should be on CPU") + self.assertTrue(statuses[busy_tid] & THREAD_STATUS_HAS_GIL, "Busy should have GIL") + + finally: + for client_socket in client_sockets: + client_socket.close() + p.terminate() + p.wait(timeout=SHORT_TIMEOUT) + p.stdout.close() + p.stderr.close() if __name__ == "__main__": diff --git a/Lib/test/test_profiling/test_sampling_profiler.py b/Lib/test/test_profiling/test_sampling_profiler.py index 0ba6799a1ce5ba..ae9bf3ef2e50e4 100644 --- a/Lib/test/test_profiling/test_sampling_profiler.py +++ b/Lib/test/test_profiling/test_sampling_profiler.py @@ -63,12 +63,14 @@ def __repr__(self): class MockThreadInfo: """Mock ThreadInfo for testing since the real one isn't accessible.""" - def __init__(self, thread_id, frame_info): + def __init__(self, thread_id, frame_info, status=0, gc_collecting=False): # Default to THREAD_STATE_RUNNING (0) self.thread_id = thread_id self.frame_info = frame_info + self.status = status + self.gc_collecting = gc_collecting def __repr__(self): - return f"MockThreadInfo(thread_id={self.thread_id}, frame_info={self.frame_info})" + return f"MockThreadInfo(thread_id={self.thread_id}, frame_info={self.frame_info}, status={self.status}, gc_collecting={self.gc_collecting})" class MockInterpreterInfo: @@ -674,6 +676,97 @@ def test_gecko_collector_export(self): self.assertIn("func2", string_array) self.assertIn("other_func", string_array) + def test_gecko_collector_markers(self): + """Test Gecko profile markers for GIL and CPU state tracking.""" + try: + from _remote_debugging import THREAD_STATUS_HAS_GIL, THREAD_STATUS_ON_CPU, THREAD_STATUS_GIL_REQUESTED + except ImportError: + THREAD_STATUS_HAS_GIL = (1 << 0) + THREAD_STATUS_ON_CPU = (1 << 1) + THREAD_STATUS_GIL_REQUESTED = (1 << 3) + + collector = GeckoCollector() + + # Status combinations for different thread states + HAS_GIL_ON_CPU = THREAD_STATUS_HAS_GIL | THREAD_STATUS_ON_CPU # Running Python code + NO_GIL_ON_CPU = THREAD_STATUS_ON_CPU # Running native code + WAITING_FOR_GIL = THREAD_STATUS_GIL_REQUESTED # Waiting for GIL + + # Simulate thread state transitions + collector.collect([ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [("test.py", 10, "python_func")], status=HAS_GIL_ON_CPU) + ]) + ]) + + collector.collect([ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [("test.py", 15, "wait_func")], status=WAITING_FOR_GIL) + ]) + ]) + + collector.collect([ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [("test.py", 20, "python_func2")], status=HAS_GIL_ON_CPU) + ]) + ]) + + collector.collect([ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [("native.c", 100, "native_func")], status=NO_GIL_ON_CPU) + ]) + ]) + + profile_data = collector._build_profile() + + # Verify we have threads with markers + self.assertIn("threads", profile_data) + self.assertEqual(len(profile_data["threads"]), 1) + thread_data = profile_data["threads"][0] + + # Check markers exist + self.assertIn("markers", thread_data) + markers = thread_data["markers"] + + # Should have marker arrays + self.assertIn("name", markers) + self.assertIn("startTime", markers) + self.assertIn("endTime", markers) + self.assertIn("category", markers) + self.assertGreater(markers["length"], 0, "Should have generated markers") + + # Get marker names from string table + string_array = profile_data["shared"]["stringArray"] + marker_names = [string_array[idx] for idx in markers["name"]] + + # Verify we have different marker types + marker_name_set = set(marker_names) + + # Should have "Has GIL" markers (when thread had GIL) + self.assertIn("Has GIL", marker_name_set, "Should have 'Has GIL' markers") + + # Should have "No GIL" markers (when thread didn't have GIL) + self.assertIn("No GIL", marker_name_set, "Should have 'No GIL' markers") + + # Should have "On CPU" markers (when thread was on CPU) + self.assertIn("On CPU", marker_name_set, "Should have 'On CPU' markers") + + # Should have "Waiting for GIL" markers (when thread was waiting) + self.assertIn("Waiting for GIL", marker_name_set, "Should have 'Waiting for GIL' markers") + + # Verify marker structure + for i in range(markers["length"]): + # All markers should be interval markers (phase = 1) + self.assertEqual(markers["phase"][i], 1, f"Marker {i} should be interval marker") + + # All markers should have valid time range + start_time = markers["startTime"][i] + end_time = markers["endTime"][i] + self.assertLessEqual(start_time, end_time, f"Marker {i} should have valid time range") + + # All markers should have valid category + self.assertGreaterEqual(markers["category"][i], 0, f"Marker {i} should have valid category") + def test_pstats_collector_export(self): collector = PstatsCollector( sample_interval_usec=1000000 @@ -2625,19 +2718,30 @@ def test_mode_validation(self): def test_frames_filtered_with_skip_idle(self): """Test that frames are actually filtered when skip_idle=True.""" + # Import thread status flags + try: + from _remote_debugging import THREAD_STATUS_HAS_GIL, THREAD_STATUS_ON_CPU + except ImportError: + THREAD_STATUS_HAS_GIL = (1 << 0) + THREAD_STATUS_ON_CPU = (1 << 1) + # Create mock frames with different thread statuses class MockThreadInfoWithStatus: def __init__(self, thread_id, frame_info, status): self.thread_id = thread_id self.frame_info = frame_info self.status = status + self.gc_collecting = False + + # Create test data: active thread (HAS_GIL | ON_CPU), idle thread (neither), and another active thread + ACTIVE_STATUS = THREAD_STATUS_HAS_GIL | THREAD_STATUS_ON_CPU # Has GIL and on CPU + IDLE_STATUS = 0 # Neither has GIL nor on CPU - # Create test data: running thread, idle thread, and another running thread test_frames = [ MockInterpreterInfo(0, [ - MockThreadInfoWithStatus(1, [MockFrameInfo("active1.py", 10, "active_func1")], 0), # RUNNING - MockThreadInfoWithStatus(2, [MockFrameInfo("idle.py", 20, "idle_func")], 1), # IDLE - MockThreadInfoWithStatus(3, [MockFrameInfo("active2.py", 30, "active_func2")], 0), # RUNNING + MockThreadInfoWithStatus(1, [MockFrameInfo("active1.py", 10, "active_func1")], ACTIVE_STATUS), + MockThreadInfoWithStatus(2, [MockFrameInfo("idle.py", 20, "idle_func")], IDLE_STATUS), + MockThreadInfoWithStatus(3, [MockFrameInfo("active2.py", 30, "active_func2")], ACTIVE_STATUS), ]) ] diff --git a/Modules/_remote_debugging_module.c b/Modules/_remote_debugging_module.c index c6ced39c70cdb3..d190b3c9fafa76 100644 --- a/Modules/_remote_debugging_module.c +++ b/Modules/_remote_debugging_module.c @@ -11,6 +11,7 @@ * HEADERS AND INCLUDES * ============================================================================ */ +#include #include #include #include @@ -81,6 +82,8 @@ typedef enum _WIN32_THREADSTATE { #define SIZEOF_TYPE_OBJ sizeof(PyTypeObject) #define SIZEOF_UNICODE_OBJ sizeof(PyUnicodeObject) #define SIZEOF_LONG_OBJ sizeof(PyLongObject) +#define SIZEOF_GC_RUNTIME_STATE sizeof(struct _gc_runtime_state) +#define SIZEOF_INTERPRETER_STATE sizeof(PyInterpreterState) // Calculate the minimum buffer size needed to read interpreter state fields // We need to read code_object_generation and potentially tlbc_generation @@ -178,8 +181,9 @@ static PyStructSequence_Desc CoroInfo_desc = { // ThreadInfo structseq type - replaces 2-tuple (thread_id, frame_info) static PyStructSequence_Field ThreadInfo_fields[] = { {"thread_id", "Thread ID"}, - {"status", "Thread status"}, + {"status", "Thread status (flags: HAS_GIL, ON_CPU, UNKNOWN or legacy enum)"}, {"frame_info", "Frame information"}, + {"gc_collecting", "Whether GC is collecting (interpreter-level)"}, {NULL} }; @@ -187,7 +191,7 @@ static PyStructSequence_Desc ThreadInfo_desc = { "_remote_debugging.ThreadInfo", "Information about a thread", ThreadInfo_fields, - 2 + 3 }; // InterpreterInfo structseq type - replaces 2-tuple (interpreter_id, thread_list) @@ -247,9 +251,16 @@ enum _ThreadState { enum _ProfilingMode { PROFILING_MODE_WALL = 0, PROFILING_MODE_CPU = 1, - PROFILING_MODE_GIL = 2 + PROFILING_MODE_GIL = 2, + PROFILING_MODE_ALL = 3 // Combines GIL + CPU checks }; +// Thread status flags (can be combined) +#define THREAD_STATUS_HAS_GIL (1 << 0) // Thread has the GIL +#define THREAD_STATUS_ON_CPU (1 << 1) // Thread is running on CPU +#define THREAD_STATUS_UNKNOWN (1 << 2) // Status could not be determined +#define THREAD_STATUS_GIL_REQUESTED (1 << 3) // Thread is waiting for the GIL + typedef struct { PyObject_HEAD proc_handle_t handle; @@ -2650,34 +2661,70 @@ unwind_stack_for_thread( long tid = GET_MEMBER(long, ts, unwinder->debug_offsets.thread_state.native_thread_id); - // Calculate thread status based on mode - int status = THREAD_STATE_UNKNOWN; - if (unwinder->mode == PROFILING_MODE_CPU) { - long pthread_id = GET_MEMBER(long, ts, unwinder->debug_offsets.thread_state.thread_id); - status = get_thread_status(unwinder, tid, pthread_id); - if (status == -1) { - PyErr_Print(); - PyErr_SetString(PyExc_RuntimeError, "Failed to get thread status"); - goto error; - } - } else if (unwinder->mode == PROFILING_MODE_GIL) { + // Read GC collecting state from the interpreter (before any skip checks) + uintptr_t interp_addr = GET_MEMBER(uintptr_t, ts, unwinder->debug_offsets.thread_state.interp); + + // Read the GC runtime state from the interpreter state + uintptr_t gc_addr = interp_addr + unwinder->debug_offsets.interpreter_state.gc; + char gc_state[SIZEOF_GC_RUNTIME_STATE]; + if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, gc_addr, unwinder->debug_offsets.gc.size, gc_state) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read GC state"); + goto error; + } + + int gc_collecting = GET_MEMBER(int, gc_state, unwinder->debug_offsets.gc.collecting); + + // Calculate thread status using flags (always) + int status_flags = 0; + + // Check GIL status + int has_gil = 0; + int gil_requested = 0; #ifdef Py_GIL_DISABLED - // All threads are considered running in free threading builds if they have a thread state attached - int active = GET_MEMBER(_thread_status, ts, unwinder->debug_offsets.thread_state.status).active; - status = active ? THREAD_STATE_RUNNING : THREAD_STATE_GIL_WAIT; + int active = GET_MEMBER(_thread_status, ts, unwinder->debug_offsets.thread_state.status).active; + has_gil = active; #else - status = (*current_tstate == gil_holder_tstate) ? THREAD_STATE_RUNNING : THREAD_STATE_GIL_WAIT; + // Read holds_gil directly from thread state + has_gil = GET_MEMBER(int, ts, unwinder->debug_offsets.thread_state.holds_gil); + + // Check if thread is actively requesting the GIL + if (unwinder->debug_offsets.thread_state.gil_requested != 0) { + gil_requested = GET_MEMBER(int, ts, unwinder->debug_offsets.thread_state.gil_requested); + } + + // Set GIL_REQUESTED flag if thread is waiting + if (!has_gil && gil_requested) { + status_flags |= THREAD_STATUS_GIL_REQUESTED; + } #endif - } else { - // PROFILING_MODE_WALL - all threads are considered running - status = THREAD_STATE_RUNNING; + if (has_gil) { + status_flags |= THREAD_STATUS_HAS_GIL; + } + + // Assert that we never have both HAS_GIL and GIL_REQUESTED set at the same time + // This would indicate a race condition in the GIL state tracking + assert(!(has_gil && gil_requested)); + + // Check CPU status + long pthread_id = GET_MEMBER(long, ts, unwinder->debug_offsets.thread_state.thread_id); + int cpu_status = get_thread_status(unwinder, tid, pthread_id); + if (cpu_status == -1) { + status_flags |= THREAD_STATUS_UNKNOWN; + } else if (cpu_status == THREAD_STATE_RUNNING) { + status_flags |= THREAD_STATUS_ON_CPU; } // Check if we should skip this thread based on mode int should_skip = 0; - if (unwinder->skip_non_matching_threads && status != THREAD_STATE_RUNNING && - (unwinder->mode == PROFILING_MODE_CPU || unwinder->mode == PROFILING_MODE_GIL)) { - should_skip = 1; + if (unwinder->skip_non_matching_threads) { + if (unwinder->mode == PROFILING_MODE_CPU) { + // Skip if not on CPU + should_skip = !(status_flags & THREAD_STATUS_ON_CPU); + } else if (unwinder->mode == PROFILING_MODE_GIL) { + // Skip if doesn't have GIL + should_skip = !(status_flags & THREAD_STATUS_HAS_GIL); + } + // PROFILING_MODE_WALL and PROFILING_MODE_ALL never skip } if (should_skip) { @@ -2719,16 +2766,25 @@ unwind_stack_for_thread( goto error; } - PyObject *py_status = PyLong_FromLong(status); + // Always use status_flags + PyObject *py_status = PyLong_FromLong(status_flags); if (py_status == NULL) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create thread status"); goto error; } - PyErr_Print(); + PyObject *py_gc_collecting = PyBool_FromLong(gc_collecting); + if (py_gc_collecting == NULL) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create gc_collecting"); + Py_DECREF(py_status); + goto error; + } + + // py_status contains status flags (bitfield) PyStructSequence_SetItem(result, 0, thread_id); PyStructSequence_SetItem(result, 1, py_status); // Steals reference PyStructSequence_SetItem(result, 2, frame_info); // Steals reference + PyStructSequence_SetItem(result, 3, py_gc_collecting); // Steals reference cleanup_stack_chunks(&chunks); return result; @@ -3401,6 +3457,21 @@ _remote_debugging_exec(PyObject *m) if (rc < 0) { return -1; } + + // Add thread status flag constants + if (PyModule_AddIntConstant(m, "THREAD_STATUS_HAS_GIL", THREAD_STATUS_HAS_GIL) < 0) { + return -1; + } + if (PyModule_AddIntConstant(m, "THREAD_STATUS_ON_CPU", THREAD_STATUS_ON_CPU) < 0) { + return -1; + } + if (PyModule_AddIntConstant(m, "THREAD_STATUS_UNKNOWN", THREAD_STATUS_UNKNOWN) < 0) { + return -1; + } + if (PyModule_AddIntConstant(m, "THREAD_STATUS_GIL_REQUESTED", THREAD_STATUS_GIL_REQUESTED) < 0) { + return -1; + } + if (RemoteDebugging_InitState(st) < 0) { return -1; } diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index 9b6506ac3326b3..f6ada3892f801d 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -207,6 +207,7 @@ drop_gil_impl(PyThreadState *tstate, struct _gil_runtime_state *gil) _Py_atomic_store_int_relaxed(&gil->locked, 0); if (tstate != NULL) { tstate->holds_gil = 0; + tstate->gil_requested = 0; } COND_SIGNAL(gil->cond); MUTEX_UNLOCK(gil->mutex); @@ -320,6 +321,8 @@ take_gil(PyThreadState *tstate) MUTEX_LOCK(gil->mutex); + tstate->gil_requested = 1; + int drop_requested = 0; while (_Py_atomic_load_int_relaxed(&gil->locked)) { unsigned long saved_switchnum = gil->switch_number; @@ -407,6 +410,7 @@ take_gil(PyThreadState *tstate) } assert(_PyThreadState_CheckConsistency(tstate)); + tstate->gil_requested = 0; tstate->holds_gil = 1; _Py_unset_eval_breaker_bit(tstate, _PY_GIL_DROP_REQUEST_BIT); update_eval_breaker_for_thread(interp, tstate); From 336366fd7ca61858572fdb78e2bd79014b215f19 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Mon, 17 Nov 2025 05:39:00 -0800 Subject: [PATCH 075/638] GH-140643: Add `` and `` frames to the sampling profiler (#141108) - Introduce a new field in the GC state to store the frame that initiated garbage collection. - Update RemoteUnwinder to include options for including "" and "" frames in the stack trace. - Modify the sampling profiler to accept parameters for controlling the inclusion of native and GC frames. - Enhance the stack collector to properly format and append these frames during profiling. - Add tests to verify the correct behavior of the profiler with respect to native and GC frames, including options to exclude them. Co-authored-by: Pablo Galindo Salgado --- Doc/library/profile.rst | 12 +- Include/internal/pycore_debug_offsets.h | 2 + .../pycore_global_objects_fini_generated.h | 4 + Include/internal/pycore_global_strings.h | 4 + Include/internal/pycore_interp_structs.h | 3 + Include/internal/pycore_interpframe_structs.h | 1 - .../internal/pycore_runtime_init_generated.h | 4 + .../internal/pycore_unicodeobject_generated.h | 16 ++ Lib/profiling/sampling/flamegraph.js | 28 ++- Lib/profiling/sampling/sample.py | 26 ++- Lib/profiling/sampling/stack_collector.py | 18 +- Lib/test/test_external_inspection.py | 2 + .../test_profiling/test_sampling_profiler.py | 208 +++++++++++++++++- ...-11-05-19-50-37.gh-issue-140643.QCEOqG.rst | 3 + Modules/_remote_debugging_module.c | 166 ++++++++++---- Modules/clinic/_remote_debugging_module.c.h | 46 +++- Python/gc.c | 2 + Python/gc_free_threading.c | 2 + 18 files changed, 463 insertions(+), 84 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-05-19-50-37.gh-issue-140643.QCEOqG.rst diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index faf8079db3ddd8..5bf36b13c6d789 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -265,6 +265,14 @@ Profile with real-time sampling statistics:: Sample all threads in the process instead of just the main thread +.. option:: --native + + Include artificial ```` frames to denote calls to non-Python code. + +.. option:: --no-gc + + Don't include artificial ```` frames to denote active garbage collection. + .. option:: --realtime-stats Print real-time sampling statistics during profiling @@ -349,7 +357,7 @@ This section documents the programmatic interface for the :mod:`!profiling.sampl For command-line usage, see :ref:`sampling-profiler-cli`. For conceptual information about statistical profiling, see :ref:`statistical-profiling` -.. function:: sample(pid, *, sort=2, sample_interval_usec=100, duration_sec=10, filename=None, all_threads=False, limit=None, show_summary=True, output_format="pstats", realtime_stats=False) +.. function:: sample(pid, *, sort=2, sample_interval_usec=100, duration_sec=10, filename=None, all_threads=False, limit=None, show_summary=True, output_format="pstats", realtime_stats=False, native=False, gc=True) Sample a Python process and generate profiling data. @@ -367,6 +375,8 @@ about statistical profiling, see :ref:`statistical-profiling` :param bool show_summary: Whether to show summary statistics (default: True) :param str output_format: Output format - 'pstats' or 'collapsed' (default: 'pstats') :param bool realtime_stats: Whether to display real-time statistics (default: False) + :param bool native: Whether to include ```` frames (default: False) + :param bool gc: Whether to include ```` frames (default: True) :raises ValueError: If output_format is not 'pstats' or 'collapsed' diff --git a/Include/internal/pycore_debug_offsets.h b/Include/internal/pycore_debug_offsets.h index f6d50bf5df7a9e..0f17bf17f82656 100644 --- a/Include/internal/pycore_debug_offsets.h +++ b/Include/internal/pycore_debug_offsets.h @@ -212,6 +212,7 @@ typedef struct _Py_DebugOffsets { struct _gc { uint64_t size; uint64_t collecting; + uint64_t frame; } gc; // Generator object offset; @@ -355,6 +356,7 @@ typedef struct _Py_DebugOffsets { .gc = { \ .size = sizeof(struct _gc_runtime_state), \ .collecting = offsetof(struct _gc_runtime_state, collecting), \ + .frame = offsetof(struct _gc_runtime_state, frame), \ }, \ .gen_object = { \ .size = sizeof(PyGenObject), \ diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 92ded14891a101..ecef4364cc32df 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1326,10 +1326,12 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dot_locals)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(empty)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(format)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(gc)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(generic_base)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(json_decoder)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(kwdefaults)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(list_err)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(native)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(str_replace_inf)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(type_params)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(utf_8)); @@ -1763,6 +1765,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(fullerror)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(func)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(future)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(gc)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(generation)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(get)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(get_debug)); @@ -1906,6 +1909,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(name_from)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(namespace_separator)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(namespaces)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(native)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ndigits)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(nested)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(new_file_name)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index cd21b0847b7cdd..4dd73291df4513 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -46,10 +46,12 @@ struct _Py_global_strings { STRUCT_FOR_STR(dot_locals, ".") STRUCT_FOR_STR(empty, "") STRUCT_FOR_STR(format, ".format") + STRUCT_FOR_STR(gc, "") STRUCT_FOR_STR(generic_base, ".generic_base") STRUCT_FOR_STR(json_decoder, "json.decoder") STRUCT_FOR_STR(kwdefaults, ".kwdefaults") STRUCT_FOR_STR(list_err, "list index out of range") + STRUCT_FOR_STR(native, "") STRUCT_FOR_STR(str_replace_inf, "1e309") STRUCT_FOR_STR(type_params, ".type_params") STRUCT_FOR_STR(utf_8, "utf-8") @@ -486,6 +488,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(fullerror) STRUCT_FOR_ID(func) STRUCT_FOR_ID(future) + STRUCT_FOR_ID(gc) STRUCT_FOR_ID(generation) STRUCT_FOR_ID(get) STRUCT_FOR_ID(get_debug) @@ -629,6 +632,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(name_from) STRUCT_FOR_ID(namespace_separator) STRUCT_FOR_ID(namespaces) + STRUCT_FOR_ID(native) STRUCT_FOR_ID(ndigits) STRUCT_FOR_ID(nested) STRUCT_FOR_ID(new_file_name) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 9e4504479cd9f0..f861d3abd96d48 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -212,6 +212,9 @@ struct _gc_runtime_state { struct gc_generation_stats generation_stats[NUM_GENERATIONS]; /* true if we are currently running the collector */ int collecting; + // The frame that started the current collection. It might be NULL even when + // collecting (if no Python frame is running): + _PyInterpreterFrame *frame; /* list of uncollectable objects */ PyObject *garbage; /* a list of callbacks to be invoked when collection is performed */ diff --git a/Include/internal/pycore_interpframe_structs.h b/Include/internal/pycore_interpframe_structs.h index 835b8e58194863..38510685f4093c 100644 --- a/Include/internal/pycore_interpframe_structs.h +++ b/Include/internal/pycore_interpframe_structs.h @@ -24,7 +24,6 @@ enum _frameowner { FRAME_OWNED_BY_GENERATOR = 1, FRAME_OWNED_BY_FRAME_OBJECT = 2, FRAME_OWNED_BY_INTERPRETER = 3, - FRAME_OWNED_BY_CSTACK = 4, }; struct _PyInterpreterFrame { diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 50d82d0a365037..08f8d0e59d12e6 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1321,10 +1321,12 @@ extern "C" { INIT_STR(dot_locals, "."), \ INIT_STR(empty, ""), \ INIT_STR(format, ".format"), \ + INIT_STR(gc, ""), \ INIT_STR(generic_base, ".generic_base"), \ INIT_STR(json_decoder, "json.decoder"), \ INIT_STR(kwdefaults, ".kwdefaults"), \ INIT_STR(list_err, "list index out of range"), \ + INIT_STR(native, ""), \ INIT_STR(str_replace_inf, "1e309"), \ INIT_STR(type_params, ".type_params"), \ INIT_STR(utf_8, "utf-8"), \ @@ -1761,6 +1763,7 @@ extern "C" { INIT_ID(fullerror), \ INIT_ID(func), \ INIT_ID(future), \ + INIT_ID(gc), \ INIT_ID(generation), \ INIT_ID(get), \ INIT_ID(get_debug), \ @@ -1904,6 +1907,7 @@ extern "C" { INIT_ID(name_from), \ INIT_ID(namespace_separator), \ INIT_ID(namespaces), \ + INIT_ID(native), \ INIT_ID(ndigits), \ INIT_ID(nested), \ INIT_ID(new_file_name), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index b4d920154b6e83..b1e57126b92d26 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1732,6 +1732,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(gc); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(generation); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2304,6 +2308,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(native); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(ndigits); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -3236,6 +3244,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(gc); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_STR(anon_null); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -3260,6 +3272,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_STR(native); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_STR(anon_setcomp); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Lib/profiling/sampling/flamegraph.js b/Lib/profiling/sampling/flamegraph.js index 95ad7ca6184ac6..670ca22d442e2b 100644 --- a/Lib/profiling/sampling/flamegraph.js +++ b/Lib/profiling/sampling/flamegraph.js @@ -151,17 +151,22 @@ function createPythonTooltip(data) { const funcname = resolveString(d.data.funcname) || resolveString(d.data.name); const filename = resolveString(d.data.filename) || ""; + // Don't show file location for special frames like and + const isSpecialFrame = filename === "~"; + const fileLocationHTML = isSpecialFrame ? "" : ` +
+ ${filename}${d.data.lineno ? ":" + d.data.lineno : ""} +
`; + const tooltipHTML = `
${funcname}
-
- ${filename}${d.data.lineno ? ":" + d.data.lineno : ""} -
+ ${fileLocationHTML}
Execution Time: @@ -474,14 +479,23 @@ function populateStats(data) { if (i < hotSpots.length && hotSpots[i]) { const hotspot = hotSpots[i]; const filename = hotspot.filename || 'unknown'; - const basename = filename !== 'unknown' ? filename.split('/').pop() : 'unknown'; const lineno = hotspot.lineno ?? '?'; let funcDisplay = hotspot.funcname || 'unknown'; if (funcDisplay.length > 35) { funcDisplay = funcDisplay.substring(0, 32) + '...'; } - document.getElementById(`hotspot-file-${num}`).textContent = `${basename}:${lineno}`; + // Don't show file:line for special frames like and + const isSpecialFrame = filename === '~' && (lineno === 0 || lineno === '?'); + let fileDisplay; + if (isSpecialFrame) { + fileDisplay = '--'; + } else { + const basename = filename !== 'unknown' ? filename.split('/').pop() : 'unknown'; + fileDisplay = `${basename}:${lineno}`; + } + + document.getElementById(`hotspot-file-${num}`).textContent = fileDisplay; document.getElementById(`hotspot-func-${num}`).textContent = funcDisplay; document.getElementById(`hotspot-detail-${num}`).textContent = `${hotspot.directPercent.toFixed(1)}% samples (${hotspot.directSamples.toLocaleString()})`; } else { diff --git a/Lib/profiling/sampling/sample.py b/Lib/profiling/sampling/sample.py index 5ca68911d8a482..713931a639dccb 100644 --- a/Lib/profiling/sampling/sample.py +++ b/Lib/profiling/sampling/sample.py @@ -137,19 +137,19 @@ def _run_with_sync(original_cmd): class SampleProfiler: - def __init__(self, pid, sample_interval_usec, all_threads, *, mode=PROFILING_MODE_WALL, skip_non_matching_threads=True): + def __init__(self, pid, sample_interval_usec, all_threads, *, mode=PROFILING_MODE_WALL, native=False, gc=True, skip_non_matching_threads=True): self.pid = pid self.sample_interval_usec = sample_interval_usec self.all_threads = all_threads if _FREE_THREADED_BUILD: self.unwinder = _remote_debugging.RemoteUnwinder( - self.pid, all_threads=self.all_threads, mode=mode, + self.pid, all_threads=self.all_threads, mode=mode, native=native, gc=gc, skip_non_matching_threads=skip_non_matching_threads ) else: only_active_threads = bool(self.all_threads) self.unwinder = _remote_debugging.RemoteUnwinder( - self.pid, only_active_thread=only_active_threads, mode=mode, + self.pid, only_active_thread=only_active_threads, mode=mode, native=native, gc=gc, skip_non_matching_threads=skip_non_matching_threads ) # Track sample intervals and total sample count @@ -616,6 +616,8 @@ def sample( output_format="pstats", realtime_stats=False, mode=PROFILING_MODE_WALL, + native=False, + gc=True, ): # PROFILING_MODE_ALL implies no skipping at all if mode == PROFILING_MODE_ALL: @@ -627,7 +629,7 @@ def sample( skip_idle = mode != PROFILING_MODE_WALL profiler = SampleProfiler( - pid, sample_interval_usec, all_threads=all_threads, mode=mode, + pid, sample_interval_usec, all_threads=all_threads, mode=mode, native=native, gc=gc, skip_non_matching_threads=skip_non_matching_threads ) profiler.realtime_stats = realtime_stats @@ -717,6 +719,8 @@ def wait_for_process_and_sample(pid, sort_value, args): output_format=args.format, realtime_stats=args.realtime_stats, mode=mode, + native=args.native, + gc=args.gc, ) @@ -767,9 +771,19 @@ def main(): sampling_group.add_argument( "--realtime-stats", action="store_true", - default=False, help="Print real-time sampling statistics (Hz, mean, min, max, stdev) during profiling", ) + sampling_group.add_argument( + "--native", + action="store_true", + help="Include artificial \"\" frames to denote calls to non-Python code.", + ) + sampling_group.add_argument( + "--no-gc", + action="store_false", + dest="gc", + help="Don't include artificial \"\" frames to denote active garbage collection.", + ) # Mode options mode_group = parser.add_argument_group("Mode options") @@ -934,6 +948,8 @@ def main(): output_format=args.format, realtime_stats=args.realtime_stats, mode=mode, + native=args.native, + gc=args.gc, ) elif args.module or args.args: if args.module: diff --git a/Lib/profiling/sampling/stack_collector.py b/Lib/profiling/sampling/stack_collector.py index bc38151e067989..1436811976a16e 100644 --- a/Lib/profiling/sampling/stack_collector.py +++ b/Lib/profiling/sampling/stack_collector.py @@ -36,10 +36,16 @@ def process_frames(self, frames, thread_id): def export(self, filename): lines = [] for (call_tree, thread_id), count in self.stack_counter.items(): - stack_str = ";".join( - f"{os.path.basename(f[0])}:{f[2]}:{f[1]}" for f in call_tree - ) - lines.append((f"tid:{thread_id};{stack_str}", count)) + parts = [f"tid:{thread_id}"] + for file, line, func in call_tree: + # This is what pstats does for "special" frames: + if file == "~" and line == 0: + part = func + else: + part = f"{os.path.basename(file)}:{func}:{line}" + parts.append(part) + stack_str = ";".join(parts) + lines.append((stack_str, count)) lines.sort(key=lambda x: (-x[1], x[0])) @@ -98,6 +104,10 @@ def export(self, filename): def _format_function_name(func): filename, lineno, funcname = func + # Special frames like and should not show file:line + if filename == "~" and lineno == 0: + return funcname + if len(filename) > 50: parts = filename.split("/") if len(parts) > 2: diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 60e5000cd72a32..7decd8f32d5a2b 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -159,6 +159,8 @@ def foo(): FrameInfo([script_name, 12, "baz"]), FrameInfo([script_name, 9, "bar"]), FrameInfo([threading.__file__, ANY, "Thread.run"]), + FrameInfo([threading.__file__, ANY, "Thread._bootstrap_inner"]), + FrameInfo([threading.__file__, ANY, "Thread._bootstrap"]), ] # Is possible that there are more threads, so we check that the # expected stack traces are in the result (looking at you Windows!) diff --git a/Lib/test/test_profiling/test_sampling_profiler.py b/Lib/test/test_profiling/test_sampling_profiler.py index ae9bf3ef2e50e4..a24dbb55cd7bab 100644 --- a/Lib/test/test_profiling/test_sampling_profiler.py +++ b/Lib/test/test_profiling/test_sampling_profiler.py @@ -2025,7 +2025,6 @@ def test_sample_target_script(self): # Should see some of our test functions self.assertIn("slow_fibonacci", output) - def test_sample_target_module(self): tempdir = tempfile.TemporaryDirectory(delete=False) self.addCleanup(lambda x: shutil.rmtree(x), tempdir.name) @@ -2264,7 +2263,9 @@ def test_cli_module_argument_parsing(self): show_summary=True, output_format="pstats", realtime_stats=False, - mode=0 + mode=0, + native=False, + gc=True, ) @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") @@ -2292,7 +2293,9 @@ def test_cli_module_with_arguments(self): show_summary=True, output_format="pstats", realtime_stats=False, - mode=0 + mode=0, + native=False, + gc=True, ) @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") @@ -2320,7 +2323,9 @@ def test_cli_script_argument_parsing(self): show_summary=True, output_format="pstats", realtime_stats=False, - mode=0 + mode=0, + native=False, + gc=True, ) @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") @@ -2420,7 +2425,9 @@ def test_cli_module_with_profiler_options(self): show_summary=True, output_format="pstats", realtime_stats=False, - mode=0 + mode=0, + native=False, + gc=True, ) @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") @@ -2454,7 +2461,9 @@ def test_cli_script_with_profiler_options(self): show_summary=True, output_format="collapsed", realtime_stats=False, - mode=0 + mode=0, + native=False, + gc=True, ) def test_cli_empty_module_name(self): @@ -2666,7 +2675,9 @@ def test_argument_parsing_basic(self): show_summary=True, output_format="pstats", realtime_stats=False, - mode=0 + mode=0, + native=False, + gc=True, ) def test_sort_options(self): @@ -3121,6 +3132,187 @@ def test_parse_mode_function(self): @requires_subprocess() @skip_if_not_supported +class TestGCFrameTracking(unittest.TestCase): + """Tests for GC frame tracking in the sampling profiler.""" + + @classmethod + def setUpClass(cls): + """Create a static test script with GC frames and CPU-intensive work.""" + cls.gc_test_script = ''' +import gc + +class ExpensiveGarbage: + """Class that triggers GC with expensive finalizer (callback).""" + def __init__(self): + self.cycle = self + + def __del__(self): + # CPU-intensive work in the finalizer callback + result = 0 + for i in range(100000): + result += i * i + if i % 1000 == 0: + result = result % 1000000 + +def main_loop(): + """Main loop that triggers GC with expensive callback.""" + while True: + ExpensiveGarbage() + gc.collect() + +if __name__ == "__main__": + main_loop() +''' + + def test_gc_frames_enabled(self): + """Test that GC frames appear when gc tracking is enabled.""" + with ( + test_subprocess(self.gc_test_script) as subproc, + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + profiling.sampling.sample.sample( + subproc.process.pid, + duration_sec=1, + sample_interval_usec=5000, + show_summary=False, + native=False, + gc=True, + ) + except PermissionError: + self.skipTest("Insufficient permissions for remote profiling") + + output = captured_output.getvalue() + + # Should capture samples + self.assertIn("Captured", output) + self.assertIn("samples", output) + + # GC frames should be present + self.assertIn("", output) + + def test_gc_frames_disabled(self): + """Test that GC frames do not appear when gc tracking is disabled.""" + with ( + test_subprocess(self.gc_test_script) as subproc, + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + profiling.sampling.sample.sample( + subproc.process.pid, + duration_sec=1, + sample_interval_usec=5000, + show_summary=False, + native=False, + gc=False, + ) + except PermissionError: + self.skipTest("Insufficient permissions for remote profiling") + + output = captured_output.getvalue() + + # Should capture samples + self.assertIn("Captured", output) + self.assertIn("samples", output) + + # GC frames should NOT be present + self.assertNotIn("", output) + + +@requires_subprocess() +@skip_if_not_supported +class TestNativeFrameTracking(unittest.TestCase): + """Tests for native frame tracking in the sampling profiler.""" + + @classmethod + def setUpClass(cls): + """Create a static test script with native frames and CPU-intensive work.""" + cls.native_test_script = ''' +import operator + +def main_loop(): + while True: + # Native code in the middle of the stack: + operator.call(inner) + +def inner(): + # Python code at the top of the stack: + for _ in range(1_000_0000): + pass + +if __name__ == "__main__": + main_loop() +''' + + def test_native_frames_enabled(self): + """Test that native frames appear when native tracking is enabled.""" + collapsed_file = tempfile.NamedTemporaryFile( + suffix=".txt", delete=False + ) + self.addCleanup(close_and_unlink, collapsed_file) + + with ( + test_subprocess(self.native_test_script) as subproc, + ): + # Suppress profiler output when testing file export + with ( + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + profiling.sampling.sample.sample( + subproc.process.pid, + duration_sec=1, + filename=collapsed_file.name, + output_format="collapsed", + sample_interval_usec=1000, + native=True, + ) + except PermissionError: + self.skipTest("Insufficient permissions for remote profiling") + + # Verify file was created and contains valid data + self.assertTrue(os.path.exists(collapsed_file.name)) + self.assertGreater(os.path.getsize(collapsed_file.name), 0) + + # Check file format + with open(collapsed_file.name, "r") as f: + content = f.read() + + lines = content.strip().split("\n") + self.assertGreater(len(lines), 0) + + stacks = [line.rsplit(" ", 1)[0] for line in lines] + + # Most samples should have native code in the middle of the stack: + self.assertTrue(any(";;" in stack for stack in stacks)) + + # No samples should have native code at the top of the stack: + self.assertFalse(any(stack.endswith(";") for stack in stacks)) + + def test_native_frames_disabled(self): + """Test that native frames do not appear when native tracking is disabled.""" + with ( + test_subprocess(self.native_test_script) as subproc, + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + profiling.sampling.sample.sample( + subproc.process.pid, + duration_sec=1, + sample_interval_usec=5000, + show_summary=False, + ) + except PermissionError: + self.skipTest("Insufficient permissions for remote profiling") + output = captured_output.getvalue() + # Native frames should NOT be present: + self.assertNotIn("", output) + + class TestProcessPoolExecutorSupport(unittest.TestCase): """ Test that ProcessPoolExecutor works correctly with profiling.sampling. @@ -3161,7 +3353,5 @@ def worker(x): self.assertIn("Results: [2, 4, 6]", stdout) self.assertNotIn("Can't pickle", stderr) - - if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-05-19-50-37.gh-issue-140643.QCEOqG.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-05-19-50-37.gh-issue-140643.QCEOqG.rst new file mode 100644 index 00000000000000..e1202dd1a17aec --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-05-19-50-37.gh-issue-140643.QCEOqG.rst @@ -0,0 +1,3 @@ +Add support for ```` and ```` frames to +:mod:`!profiling.sampling` output to denote active garbage collection and +calls to native code. diff --git a/Modules/_remote_debugging_module.c b/Modules/_remote_debugging_module.c index d190b3c9fafa76..51b3c6bac02b54 100644 --- a/Modules/_remote_debugging_module.c +++ b/Modules/_remote_debugging_module.c @@ -26,8 +26,9 @@ #include "Python.h" #include // _Py_DebugOffsets #include // FRAME_SUSPENDED_YIELD_FROM -#include // FRAME_OWNED_BY_CSTACK +#include // FRAME_OWNED_BY_INTERPRETER #include // struct llist_node +#include // _PyLong_GetZero #include // Py_TAG_BITS #include "../Python/remote_debug.h" @@ -92,14 +93,16 @@ typedef enum _WIN32_THREADSTATE { #endif #ifdef Py_GIL_DISABLED -#define INTERP_STATE_MIN_SIZE MAX(MAX(MAX(offsetof(PyInterpreterState, _code_object_generation) + sizeof(uint64_t), \ - offsetof(PyInterpreterState, tlbc_indices.tlbc_generation) + sizeof(uint32_t)), \ - offsetof(PyInterpreterState, threads.head) + sizeof(void*)), \ - offsetof(PyInterpreterState, _gil.last_holder) + sizeof(PyThreadState*)) +#define INTERP_STATE_MIN_SIZE MAX(MAX(MAX(MAX(offsetof(PyInterpreterState, _code_object_generation) + sizeof(uint64_t), \ + offsetof(PyInterpreterState, tlbc_indices.tlbc_generation) + sizeof(uint32_t)), \ + offsetof(PyInterpreterState, threads.head) + sizeof(void*)), \ + offsetof(PyInterpreterState, _gil.last_holder) + sizeof(PyThreadState*)), \ + offsetof(PyInterpreterState, gc.frame) + sizeof(_PyInterpreterFrame *)) #else -#define INTERP_STATE_MIN_SIZE MAX(MAX(offsetof(PyInterpreterState, _code_object_generation) + sizeof(uint64_t), \ - offsetof(PyInterpreterState, threads.head) + sizeof(void*)), \ - offsetof(PyInterpreterState, _gil.last_holder) + sizeof(PyThreadState*)) +#define INTERP_STATE_MIN_SIZE MAX(MAX(MAX(offsetof(PyInterpreterState, _code_object_generation) + sizeof(uint64_t), \ + offsetof(PyInterpreterState, threads.head) + sizeof(void*)), \ + offsetof(PyInterpreterState, _gil.last_holder) + sizeof(PyThreadState*)), \ + offsetof(PyInterpreterState, gc.frame) + sizeof(_PyInterpreterFrame *)) #endif #define INTERP_STATE_BUFFER_SIZE MAX(INTERP_STATE_MIN_SIZE, 256) @@ -276,6 +279,8 @@ typedef struct { int only_active_thread; int mode; // Use enum _ProfilingMode values int skip_non_matching_threads; // New option to skip threads that don't match mode + int native; + int gc; RemoteDebuggingState *cached_state; // Cached module state #ifdef Py_GIL_DISABLED // TLBC cache invalidation tracking @@ -1812,6 +1817,25 @@ parse_linetable(const uintptr_t addrq, const char* linetable, int firstlineno, L * CODE OBJECT AND FRAME PARSING FUNCTIONS * ============================================================================ */ +static PyObject * +make_frame_info(RemoteUnwinderObject *unwinder, PyObject *file, PyObject *line, + PyObject *func) +{ + RemoteDebuggingState *state = RemoteDebugging_GetStateFromObject((PyObject*)unwinder); + PyObject *info = PyStructSequence_New(state->FrameInfo_Type); + if (info == NULL) { + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create FrameInfo"); + return NULL; + } + Py_INCREF(file); + Py_INCREF(line); + Py_INCREF(func); + PyStructSequence_SetItem(info, 0, file); + PyStructSequence_SetItem(info, 1, line); + PyStructSequence_SetItem(info, 2, func); + return info; +} + static int parse_code_object(RemoteUnwinderObject *unwinder, PyObject **result, @@ -1825,8 +1849,6 @@ parse_code_object(RemoteUnwinderObject *unwinder, PyObject *func = NULL; PyObject *file = NULL; PyObject *linetable = NULL; - PyObject *lineno = NULL; - PyObject *tuple = NULL; #ifdef Py_GIL_DISABLED // In free threading builds, code object addresses might have the low bit set @@ -1948,25 +1970,18 @@ parse_code_object(RemoteUnwinderObject *unwinder, info.lineno = -1; } - lineno = PyLong_FromLong(info.lineno); + PyObject *lineno = PyLong_FromLong(info.lineno); if (!lineno) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create line number object"); goto error; } - RemoteDebuggingState *state = RemoteDebugging_GetStateFromObject((PyObject*)unwinder); - tuple = PyStructSequence_New(state->FrameInfo_Type); + PyObject *tuple = make_frame_info(unwinder, meta->file_name, lineno, meta->func_name); + Py_DECREF(lineno); if (!tuple) { - set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create FrameInfo for code object"); goto error; } - Py_INCREF(meta->func_name); - Py_INCREF(meta->file_name); - PyStructSequence_SetItem(tuple, 0, meta->file_name); - PyStructSequence_SetItem(tuple, 1, lineno); - PyStructSequence_SetItem(tuple, 2, meta->func_name); - *result = tuple; return 0; @@ -1974,8 +1989,6 @@ parse_code_object(RemoteUnwinderObject *unwinder, Py_XDECREF(func); Py_XDECREF(file); Py_XDECREF(linetable); - Py_XDECREF(lineno); - Py_XDECREF(tuple); return -1; } @@ -2116,6 +2129,7 @@ parse_frame_from_chunks( PyObject **result, uintptr_t address, uintptr_t *previous_frame, + uintptr_t *stackpointer, StackChunkList *chunks ) { void *frame_ptr = find_frame_in_chunks(chunks, address); @@ -2126,6 +2140,7 @@ parse_frame_from_chunks( char *frame = (char *)frame_ptr; *previous_frame = GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.previous); + *stackpointer = GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.stackpointer); uintptr_t code_object = GET_MEMBER_NO_TAG(uintptr_t, frame_ptr, unwinder->debug_offsets.interpreter_frame.executable); int frame_valid = is_frame_valid(unwinder, (uintptr_t)frame, code_object); if (frame_valid != 1) { @@ -2238,8 +2253,7 @@ is_frame_valid( void* frame = (void*)frame_addr; - if (GET_MEMBER(char, frame, unwinder->debug_offsets.interpreter_frame.owner) == FRAME_OWNED_BY_CSTACK || - GET_MEMBER(char, frame, unwinder->debug_offsets.interpreter_frame.owner) == FRAME_OWNED_BY_INTERPRETER) { + if (GET_MEMBER(char, frame, unwinder->debug_offsets.interpreter_frame.owner) == FRAME_OWNED_BY_INTERPRETER) { return 0; // C frame } @@ -2458,8 +2472,9 @@ process_frame_chain( RemoteUnwinderObject *unwinder, uintptr_t initial_frame_addr, StackChunkList *chunks, - PyObject *frame_info -) { + PyObject *frame_info, + uintptr_t gc_frame) +{ uintptr_t frame_addr = initial_frame_addr; uintptr_t prev_frame_addr = 0; const size_t MAX_FRAMES = 1024; @@ -2468,6 +2483,7 @@ process_frame_chain( while ((void*)frame_addr != NULL) { PyObject *frame = NULL; uintptr_t next_frame_addr = 0; + uintptr_t stackpointer = 0; if (++frame_count > MAX_FRAMES) { PyErr_SetString(PyExc_RuntimeError, "Too many stack frames (possible infinite loop)"); @@ -2476,7 +2492,7 @@ process_frame_chain( } // Try chunks first, fallback to direct memory read - if (parse_frame_from_chunks(unwinder, &frame, frame_addr, &next_frame_addr, chunks) < 0) { + if (parse_frame_from_chunks(unwinder, &frame, frame_addr, &next_frame_addr, &stackpointer, chunks) < 0) { PyErr_Clear(); uintptr_t address_of_code_object = 0; if (parse_frame_object(unwinder, &frame, frame_addr, &address_of_code_object ,&next_frame_addr) < 0) { @@ -2484,26 +2500,63 @@ process_frame_chain( return -1; } } - - if (!frame) { - break; - } - - if (prev_frame_addr && frame_addr != prev_frame_addr) { - PyErr_Format(PyExc_RuntimeError, - "Broken frame chain: expected frame at 0x%lx, got 0x%lx", - prev_frame_addr, frame_addr); - Py_DECREF(frame); - set_exception_cause(unwinder, PyExc_RuntimeError, "Frame chain consistency check failed"); + if (frame == NULL && PyList_GET_SIZE(frame_info) == 0) { + // If the first frame is missing, the chain is broken: + const char *e = "Failed to parse initial frame in chain"; + PyErr_SetString(PyExc_RuntimeError, e); return -1; } + PyObject *extra_frame = NULL; + // This frame kicked off the current GC collection: + if (unwinder->gc && frame_addr == gc_frame) { + _Py_DECLARE_STR(gc, ""); + extra_frame = &_Py_STR(gc); + } + // Otherwise, check for native frames to insert: + else if (unwinder->native && + // We've reached an interpreter trampoline frame: + frame == NULL && + // Bottommost frame is always native, so skip that one: + next_frame_addr && + // Only suppress native frames if GC tracking is enabled and the next frame will be a GC frame: + !(unwinder->gc && next_frame_addr == gc_frame)) + { + _Py_DECLARE_STR(native, ""); + extra_frame = &_Py_STR(native); + } + if (extra_frame) { + // Use "~" as file and 0 as line, since that's what pstats uses: + PyObject *extra_frame_info = make_frame_info( + unwinder, _Py_LATIN1_CHR('~'), _PyLong_GetZero(), extra_frame); + if (extra_frame_info == NULL) { + return -1; + } + int error = PyList_Append(frame_info, extra_frame_info); + Py_DECREF(extra_frame_info); + if (error) { + const char *e = "Failed to append extra frame to frame info list"; + set_exception_cause(unwinder, PyExc_RuntimeError, e); + return -1; + } + } + if (frame) { + if (prev_frame_addr && frame_addr != prev_frame_addr) { + const char *f = "Broken frame chain: expected frame at 0x%lx, got 0x%lx"; + PyErr_Format(PyExc_RuntimeError, f, prev_frame_addr, frame_addr); + Py_DECREF(frame); + const char *e = "Frame chain consistency check failed"; + set_exception_cause(unwinder, PyExc_RuntimeError, e); + return -1; + } - if (PyList_Append(frame_info, frame) == -1) { + if (PyList_Append(frame_info, frame) == -1) { + Py_DECREF(frame); + const char *e = "Failed to append frame to frame info list"; + set_exception_cause(unwinder, PyExc_RuntimeError, e); + return -1; + } Py_DECREF(frame); - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append frame to frame info list"); - return -1; } - Py_DECREF(frame); prev_frame_addr = next_frame_addr; frame_addr = next_frame_addr; @@ -2644,7 +2697,8 @@ static PyObject* unwind_stack_for_thread( RemoteUnwinderObject *unwinder, uintptr_t *current_tstate, - uintptr_t gil_holder_tstate + uintptr_t gil_holder_tstate, + uintptr_t gc_frame ) { PyObject *frame_info = NULL; PyObject *thread_id = NULL; @@ -2746,7 +2800,7 @@ unwind_stack_for_thread( goto error; } - if (process_frame_chain(unwinder, frame_addr, &chunks, frame_info) < 0) { + if (process_frame_chain(unwinder, frame_addr, &chunks, frame_info, gc_frame) < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to process frame chain"); goto error; } @@ -2818,6 +2872,8 @@ _remote_debugging.RemoteUnwinder.__init__ mode: int = 0 debug: bool = False skip_non_matching_threads: bool = True + native: bool = False + gc: bool = False Initialize a new RemoteUnwinder object for debugging a remote Python process. @@ -2832,6 +2888,10 @@ Initialize a new RemoteUnwinder object for debugging a remote Python process. lead to the exception. skip_non_matching_threads: If True, skip threads that don't match the selected mode. If False, include all threads regardless of mode. + native: If True, include artificial "" frames to denote calls to + non-Python code. + gc: If True, include artificial "" frames to denote active garbage + collection. The RemoteUnwinder provides functionality to inspect and debug a running Python process, including examining thread states, stack frames and other runtime data. @@ -2848,8 +2908,9 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, int pid, int all_threads, int only_active_thread, int mode, int debug, - int skip_non_matching_threads) -/*[clinic end generated code: output=abf5ea5cd58bcb36 input=08fb6ace023ec3b5]*/ + int skip_non_matching_threads, + int native, int gc) +/*[clinic end generated code: output=e9eb6b4df119f6e0 input=606d099059207df2]*/ { // Validate that all_threads and only_active_thread are not both True if (all_threads && only_active_thread) { @@ -2866,6 +2927,8 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, } #endif + self->native = native; + self->gc = gc; self->debug = debug; self->only_active_thread = only_active_thread; self->mode = mode; @@ -3026,6 +3089,13 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self goto exit; } + uintptr_t gc_frame = 0; + if (self->gc) { + gc_frame = GET_MEMBER(uintptr_t, interp_state_buffer, + self->debug_offsets.interpreter_state.gc + + self->debug_offsets.gc.frame); + } + int64_t interpreter_id = GET_MEMBER(int64_t, interp_state_buffer, self->debug_offsets.interpreter_state.id); @@ -3085,7 +3155,9 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self } while (current_tstate != 0) { - PyObject* frame_info = unwind_stack_for_thread(self, ¤t_tstate, gil_holder_tstate); + PyObject* frame_info = unwind_stack_for_thread(self, ¤t_tstate, + gil_holder_tstate, + gc_frame); if (!frame_info) { // Check if this was an intentional skip due to mode-based filtering if ((self->mode == PROFILING_MODE_CPU || self->mode == PROFILING_MODE_GIL) && !PyErr_Occurred()) { diff --git a/Modules/clinic/_remote_debugging_module.c.h b/Modules/clinic/_remote_debugging_module.c.h index 7dd54e3124887b..60adb357e32e71 100644 --- a/Modules/clinic/_remote_debugging_module.c.h +++ b/Modules/clinic/_remote_debugging_module.c.h @@ -11,7 +11,8 @@ preserve PyDoc_STRVAR(_remote_debugging_RemoteUnwinder___init____doc__, "RemoteUnwinder(pid, *, all_threads=False, only_active_thread=False,\n" -" mode=0, debug=False, skip_non_matching_threads=True)\n" +" mode=0, debug=False, skip_non_matching_threads=True,\n" +" native=False, gc=False)\n" "--\n" "\n" "Initialize a new RemoteUnwinder object for debugging a remote Python process.\n" @@ -27,6 +28,10 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder___init____doc__, " lead to the exception.\n" " skip_non_matching_threads: If True, skip threads that don\'t match the selected mode.\n" " If False, include all threads regardless of mode.\n" +" native: If True, include artificial \"\" frames to denote calls to\n" +" non-Python code.\n" +" gc: If True, include artificial \"\" frames to denote active garbage\n" +" collection.\n" "\n" "The RemoteUnwinder provides functionality to inspect and debug a running Python\n" "process, including examining thread states, stack frames and other runtime data.\n" @@ -42,7 +47,8 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, int pid, int all_threads, int only_active_thread, int mode, int debug, - int skip_non_matching_threads); + int skip_non_matching_threads, + int native, int gc); static int _remote_debugging_RemoteUnwinder___init__(PyObject *self, PyObject *args, PyObject *kwargs) @@ -50,7 +56,7 @@ _remote_debugging_RemoteUnwinder___init__(PyObject *self, PyObject *args, PyObje int return_value = -1; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 6 + #define NUM_KEYWORDS 8 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -59,7 +65,7 @@ _remote_debugging_RemoteUnwinder___init__(PyObject *self, PyObject *args, PyObje } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(pid), &_Py_ID(all_threads), &_Py_ID(only_active_thread), &_Py_ID(mode), &_Py_ID(debug), &_Py_ID(skip_non_matching_threads), }, + .ob_item = { &_Py_ID(pid), &_Py_ID(all_threads), &_Py_ID(only_active_thread), &_Py_ID(mode), &_Py_ID(debug), &_Py_ID(skip_non_matching_threads), &_Py_ID(native), &_Py_ID(gc), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -68,14 +74,14 @@ _remote_debugging_RemoteUnwinder___init__(PyObject *self, PyObject *args, PyObje # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"pid", "all_threads", "only_active_thread", "mode", "debug", "skip_non_matching_threads", NULL}; + static const char * const _keywords[] = {"pid", "all_threads", "only_active_thread", "mode", "debug", "skip_non_matching_threads", "native", "gc", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "RemoteUnwinder", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[6]; + PyObject *argsbuf[8]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; @@ -85,6 +91,8 @@ _remote_debugging_RemoteUnwinder___init__(PyObject *self, PyObject *args, PyObje int mode = 0; int debug = 0; int skip_non_matching_threads = 1; + int native = 0; + int gc = 0; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -134,12 +142,30 @@ _remote_debugging_RemoteUnwinder___init__(PyObject *self, PyObject *args, PyObje goto skip_optional_kwonly; } } - skip_non_matching_threads = PyObject_IsTrue(fastargs[5]); - if (skip_non_matching_threads < 0) { + if (fastargs[5]) { + skip_non_matching_threads = PyObject_IsTrue(fastargs[5]); + if (skip_non_matching_threads < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[6]) { + native = PyObject_IsTrue(fastargs[6]); + if (native < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + gc = PyObject_IsTrue(fastargs[7]); + if (gc < 0) { goto exit; } skip_optional_kwonly: - return_value = _remote_debugging_RemoteUnwinder___init___impl((RemoteUnwinderObject *)self, pid, all_threads, only_active_thread, mode, debug, skip_non_matching_threads); + return_value = _remote_debugging_RemoteUnwinder___init___impl((RemoteUnwinderObject *)self, pid, all_threads, only_active_thread, mode, debug, skip_non_matching_threads, native, gc); exit: return return_value; @@ -321,4 +347,4 @@ _remote_debugging_RemoteUnwinder_get_async_stack_trace(PyObject *self, PyObject return return_value; } -/*[clinic end generated code: output=2caefeddf7683d32 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=99fed5c94cf36881 input=a9049054013a1b77]*/ diff --git a/Python/gc.c b/Python/gc.c index 03a5d7366ea6c9..064f9406e0a17c 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -2074,6 +2074,7 @@ _PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason) // Don't start a garbage collection if one is already in progress. return 0; } + gcstate->frame = tstate->current_frame; struct gc_collection_stats stats = { 0 }; if (reason != _Py_GC_REASON_SHUTDOWN) { @@ -2119,6 +2120,7 @@ _PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason) } #endif validate_spaces(gcstate); + gcstate->frame = NULL; _Py_atomic_store_int(&gcstate->collecting, 0); if (gcstate->debug & _PyGC_DEBUG_STATS) { diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index b183062eff7952..7724676c2426dc 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -2359,6 +2359,7 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) _Py_atomic_store_int(&gcstate->collecting, 0); return 0; } + gcstate->frame = tstate->current_frame; assert(generation >= 0 && generation < NUM_GENERATIONS); @@ -2447,6 +2448,7 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) } assert(!_PyErr_Occurred(tstate)); + gcstate->frame = NULL; _Py_atomic_store_int(&gcstate->collecting, 0); return n + m; } From f6dd9c12a8ba391cbbcc793411ac7dcfa6e01028 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Mon, 17 Nov 2025 05:41:22 -0800 Subject: [PATCH 076/638] GH-139914: Handle stack growth direction on HPPA (GH-140028) Adapted from a patch for Python 3.14 submitted to the Debian BTS by John https://bugs.debian.org/1105111#20 Co-authored-by: John David Anglin --- Include/internal/pycore_ceval.h | 8 ++++ Include/internal/pycore_pystate.h | 4 ++ Include/pyport.h | 6 +++ Lib/test/test_call.py | 9 +++- ...-10-13-13-54-19.gh-issue-139914.M-y_3E.rst | 1 + Modules/_testcapimodule.c | 4 ++ Python/ceval.c | 43 +++++++++++++++++-- configure | 13 ++++++ configure.ac | 8 ++++ pyconfig.h.in | 3 ++ 10 files changed, 94 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-13-13-54-19.gh-issue-139914.M-y_3E.rst diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 33b9fd053f70cb..47c42fccdc2376 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -217,7 +217,11 @@ extern void _PyEval_DeactivateOpCache(void); static inline int _Py_MakeRecCheck(PyThreadState *tstate) { uintptr_t here_addr = _Py_get_machine_stack_pointer(); _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; +#if _Py_STACK_GROWS_DOWN return here_addr < _tstate->c_stack_soft_limit; +#else + return here_addr > _tstate->c_stack_soft_limit; +#endif } // Export for '_json' shared extension, used via _Py_EnterRecursiveCall() @@ -249,7 +253,11 @@ static inline int _Py_ReachedRecursionLimit(PyThreadState *tstate) { uintptr_t here_addr = _Py_get_machine_stack_pointer(); _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; assert(_tstate->c_stack_hard_limit != 0); +#if _Py_STACK_GROWS_DOWN return here_addr <= _tstate->c_stack_soft_limit; +#else + return here_addr >= _tstate->c_stack_soft_limit; +#endif } static inline void _Py_LeaveRecursiveCall(void) { diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index cab458f84028e2..189a8dde9f09ed 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -331,7 +331,11 @@ _Py_RecursionLimit_GetMargin(PyThreadState *tstate) _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; assert(_tstate->c_stack_hard_limit != 0); intptr_t here_addr = _Py_get_machine_stack_pointer(); +#if _Py_STACK_GROWS_DOWN return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, here_addr - (intptr_t)_tstate->c_stack_soft_limit, _PyOS_STACK_MARGIN_SHIFT); +#else + return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, (intptr_t)_tstate->c_stack_soft_limit - here_addr, _PyOS_STACK_MARGIN_SHIFT); +#endif } #ifdef __cplusplus diff --git a/Include/pyport.h b/Include/pyport.h index e77b39026a59c1..b250f9e308f2dd 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -677,4 +677,10 @@ extern "C" { #endif +// Assume the stack grows down unless specified otherwise +#ifndef _Py_STACK_GROWS_DOWN +# define _Py_STACK_GROWS_DOWN 1 +#endif + + #endif /* Py_PYPORT_H */ diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index 31e58e825be422..f42526aee19417 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -1048,9 +1048,14 @@ def get_sp(): this_sp = _testinternalcapi.get_stack_pointer() lower_sp = _testcapi.pyobject_vectorcall(get_sp, (), ()) - self.assertLess(lower_sp, this_sp) + if _testcapi._Py_STACK_GROWS_DOWN: + self.assertLess(lower_sp, this_sp) + safe_margin = this_sp - lower_sp + else: + self.assertGreater(lower_sp, this_sp) + safe_margin = lower_sp - this_sp # Add an (arbitrary) extra 25% for safety - safe_margin = (this_sp - lower_sp) * 5 / 4 + safe_margin = safe_margin * 5 / 4 self.assertLess(safe_margin, _testinternalcapi.get_stack_margin()) @skip_on_s390x diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-13-13-54-19.gh-issue-139914.M-y_3E.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-13-13-54-19.gh-issue-139914.M-y_3E.rst new file mode 100644 index 00000000000000..7529108d5d4772 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-13-13-54-19.gh-issue-139914.M-y_3E.rst @@ -0,0 +1 @@ +Restore support for HP PA-RISC, which has an upwards-growing stack. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 22cd731d410082..c14f925b4e7632 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3359,6 +3359,10 @@ _testcapi_exec(PyObject *m) PyModule_AddObject(m, "INT64_MAX", PyLong_FromInt64(INT64_MAX)); PyModule_AddObject(m, "UINT64_MAX", PyLong_FromUInt64(UINT64_MAX)); + if (PyModule_AddIntMacro(m, _Py_STACK_GROWS_DOWN)) { + return -1; + } + if (PyModule_AddIntMacro(m, Py_single_input)) { return -1; } diff --git a/Python/ceval.c b/Python/ceval.c index 31b81a37464718..25294ebd993f6c 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -351,13 +351,21 @@ _Py_ReachedRecursionLimitWithMargin(PyThreadState *tstate, int margin_count) { uintptr_t here_addr = _Py_get_machine_stack_pointer(); _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; +#if _Py_STACK_GROWS_DOWN if (here_addr > _tstate->c_stack_soft_limit + margin_count * _PyOS_STACK_MARGIN_BYTES) { +#else + if (here_addr <= _tstate->c_stack_soft_limit - margin_count * _PyOS_STACK_MARGIN_BYTES) { +#endif return 0; } if (_tstate->c_stack_hard_limit == 0) { _Py_InitializeRecursionLimits(tstate); } +#if _Py_STACK_GROWS_DOWN return here_addr <= _tstate->c_stack_soft_limit + margin_count * _PyOS_STACK_MARGIN_BYTES; +#else + return here_addr > _tstate->c_stack_soft_limit - margin_count * _PyOS_STACK_MARGIN_BYTES; +#endif } void @@ -365,7 +373,11 @@ _Py_EnterRecursiveCallUnchecked(PyThreadState *tstate) { uintptr_t here_addr = _Py_get_machine_stack_pointer(); _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; +#if _Py_STACK_GROWS_DOWN if (here_addr < _tstate->c_stack_hard_limit) { +#else + if (here_addr > _tstate->c_stack_hard_limit) { +#endif Py_FatalError("Unchecked stack overflow."); } } @@ -496,18 +508,33 @@ tstate_set_stack(PyThreadState *tstate, #ifdef _Py_THREAD_SANITIZER // Thread sanitizer crashes if we use more than half the stack. uintptr_t stacksize = top - base; - base += stacksize / 2; +# if _Py_STACK_GROWS_DOWN + base += stacksize/2; +# else + top -= stacksize/2; +# endif #endif _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; +#if _Py_STACK_GROWS_DOWN _tstate->c_stack_top = top; _tstate->c_stack_hard_limit = base + _PyOS_STACK_MARGIN_BYTES; _tstate->c_stack_soft_limit = base + _PyOS_STACK_MARGIN_BYTES * 2; - -#ifndef NDEBUG +# ifndef NDEBUG // Sanity checks _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; assert(ts->c_stack_hard_limit <= ts->c_stack_soft_limit); assert(ts->c_stack_soft_limit < ts->c_stack_top); +# endif +#else + _tstate->c_stack_top = base; + _tstate->c_stack_hard_limit = top - _PyOS_STACK_MARGIN_BYTES; + _tstate->c_stack_soft_limit = top - _PyOS_STACK_MARGIN_BYTES * 2; +# ifndef NDEBUG + // Sanity checks + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; + assert(ts->c_stack_hard_limit >= ts->c_stack_soft_limit); + assert(ts->c_stack_soft_limit > ts->c_stack_top); +# endif #endif } @@ -568,9 +595,15 @@ _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where) uintptr_t here_addr = _Py_get_machine_stack_pointer(); assert(_tstate->c_stack_soft_limit != 0); assert(_tstate->c_stack_hard_limit != 0); +#if _Py_STACK_GROWS_DOWN if (here_addr < _tstate->c_stack_hard_limit) { /* Overflowing while handling an overflow. Give up. */ int kbytes_used = (int)(_tstate->c_stack_top - here_addr)/1024; +#else + if (here_addr > _tstate->c_stack_hard_limit) { + /* Overflowing while handling an overflow. Give up. */ + int kbytes_used = (int)(here_addr - _tstate->c_stack_top)/1024; +#endif char buffer[80]; snprintf(buffer, 80, "Unrecoverable stack overflow (used %d kB)%s", kbytes_used, where); Py_FatalError(buffer); @@ -579,7 +612,11 @@ _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where) return 0; } else { +#if _Py_STACK_GROWS_DOWN int kbytes_used = (int)(_tstate->c_stack_top - here_addr)/1024; +#else + int kbytes_used = (int)(here_addr - _tstate->c_stack_top)/1024; +#endif tstate->recursion_headroom++; _PyErr_Format(tstate, PyExc_RecursionError, "Stack overflow (used %d kB)%s", diff --git a/configure b/configure index eeb24c1d844e86..a4514f80c3af37 100755 --- a/configure +++ b/configure @@ -967,6 +967,7 @@ LDLIBRARY LIBRARY BUILDEXEEXT NO_AS_NEEDED +_Py_STACK_GROWS_DOWN MULTIARCH_CPPFLAGS PLATFORM_TRIPLET MULTIARCH @@ -7213,6 +7214,18 @@ if test x$MULTIARCH != x; then fi +# Guess C stack direction +case $host in #( + hppa*) : + _Py_STACK_GROWS_DOWN=0 ;; #( + *) : + _Py_STACK_GROWS_DOWN=1 ;; +esac + +printf "%s\n" "#define _Py_STACK_GROWS_DOWN $_Py_STACK_GROWS_DOWN" >>confdefs.h + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for PEP 11 support tier" >&5 printf %s "checking for PEP 11 support tier... " >&6; } case $host/$ac_cv_cc_name in #( diff --git a/configure.ac b/configure.ac index 92adc44da0d6fe..a059a07bec2fe4 100644 --- a/configure.ac +++ b/configure.ac @@ -1202,6 +1202,14 @@ if test x$MULTIARCH != x; then fi AC_SUBST([MULTIARCH_CPPFLAGS]) +# Guess C stack direction +AS_CASE([$host], + [hppa*], [_Py_STACK_GROWS_DOWN=0], + [_Py_STACK_GROWS_DOWN=1]) +AC_DEFINE_UNQUOTED([_Py_STACK_GROWS_DOWN], [$_Py_STACK_GROWS_DOWN], + [Define to 1 if the machine stack grows down (default); 0 if it grows up.]) +AC_SUBST([_Py_STACK_GROWS_DOWN]) + dnl Support tiers according to https://peps.python.org/pep-0011/ dnl dnl NOTE: Windows support tiers are defined in PC/pyconfig.h. diff --git a/pyconfig.h.in b/pyconfig.h.in index fb12079bafa95e..8a9f5ca8ec826d 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -2050,6 +2050,9 @@ /* HACL* library can compile SIMD256 implementations */ #undef _Py_HACL_CAN_COMPILE_VEC256 +/* Define to 1 if the machine stack grows down (default); 0 if it grows up. */ +#undef _Py_STACK_GROWS_DOWN + /* Define if you want to use tail-calling interpreters in CPython. */ #undef _Py_TAIL_CALL_INTERP From 3d148059479b28a21f8eae6abf6d1bcc91ab8cbb Mon Sep 17 00:00:00 2001 From: "R.C.M" Date: Mon, 17 Nov 2025 09:42:26 -0500 Subject: [PATCH 077/638] gh-130693: Support more options for search in tkinter.Text (GH-130848) * Add parameters nolinestop and strictlimits in the tkinter.Text.search() method. * Add the tkinter.Text.search_all() method. * Add more tests for tkinter.Text.search(). * stopindex is now only ignored if it is None. --- Doc/whatsnew/3.15.rst | 13 ++ Lib/test/test_tkinter/test_text.py | 114 +++++++++++++++++- Lib/tkinter/__init__.py | 34 +++++- ...-03-04-17-19-26.gh-issue-130693.Kv01r8.rst | 1 + 4 files changed, 154 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-03-04-17-19-26.gh-issue-130693.Kv01r8.rst diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 9393b65ed8e906..cf5bef15203b23 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -734,6 +734,19 @@ timeit :ref:`environment variables `. (Contributed by Yi Hong in :gh:`139374`.) +tkinter +------- + +* The :meth:`!tkinter.Text.search` method now supports two additional + arguments: *nolinestop* which allows the search to + continue across line boundaries; + and *strictlimits* which restricts the search to within the specified range. + (Contributed by Rihaan Meher in :gh:`130848`) + +* A new method :meth:`!tkinter.Text.search_all` has been introduced. + This method allows for searching for all matches of a pattern + using Tcl's ``-all`` and ``-overlap`` options. + (Contributed by Rihaan Meher in :gh:`130848`) types ------ diff --git a/Lib/test/test_tkinter/test_text.py b/Lib/test/test_tkinter/test_text.py index b26956930d3402..d579cca95ee2bb 100644 --- a/Lib/test/test_tkinter/test_text.py +++ b/Lib/test/test_tkinter/test_text.py @@ -34,12 +34,116 @@ def test_search(self): # Invalid text index. self.assertRaises(tkinter.TclError, text.search, '', 0) + self.assertRaises(tkinter.TclError, text.search, '', '') + self.assertRaises(tkinter.TclError, text.search, '', 'invalid') + self.assertRaises(tkinter.TclError, text.search, '', '1.0', 0) + self.assertRaises(tkinter.TclError, text.search, '', '1.0', '') + self.assertRaises(tkinter.TclError, text.search, '', '1.0', 'invalid') - # Check if we are getting the indices as strings -- you are likely - # to get Tcl_Obj under Tk 8.5 if Tkinter doesn't convert it. - text.insert('1.0', 'hi-test') - self.assertEqual(text.search('-test', '1.0', 'end'), '1.2') - self.assertEqual(text.search('test', '1.0', 'end'), '1.3') + text.insert('1.0', + 'This is a test. This is only a test.\n' + 'Another line.\n' + 'Yet another line.\n' + '64-bit') + + self.assertEqual(text.search('test', '1.0'), '1.10') + self.assertEqual(text.search('test', '1.0', 'end'), '1.10') + self.assertEqual(text.search('test', '1.0', '1.10'), '') + self.assertEqual(text.search('test', '1.11'), '1.31') + self.assertEqual(text.search('test', '1.32', 'end'), '') + self.assertEqual(text.search('test', '1.32'), '1.10') + + self.assertEqual(text.search('', '1.0'), '1.0') # empty pattern + self.assertEqual(text.search('nonexistent', '1.0'), '') + self.assertEqual(text.search('-bit', '1.0'), '4.2') # starts with a hyphen + + self.assertEqual(text.search('line', '3.0'), '3.12') + self.assertEqual(text.search('line', '3.0', forwards=True), '3.12') + self.assertEqual(text.search('line', '3.0', backwards=True), '2.8') + self.assertEqual(text.search('line', '3.0', forwards=True, backwards=True), '2.8') + + self.assertEqual(text.search('t.', '1.0'), '1.13') + self.assertEqual(text.search('t.', '1.0', exact=True), '1.13') + self.assertEqual(text.search('t.', '1.0', regexp=True), '1.10') + self.assertEqual(text.search('t.', '1.0', exact=True, regexp=True), '1.10') + + self.assertEqual(text.search('TEST', '1.0'), '') + self.assertEqual(text.search('TEST', '1.0', nocase=True), '1.10') + + self.assertEqual(text.search('.*line', '1.0', regexp=True), '2.0') + self.assertEqual(text.search('.*line', '1.0', regexp=True, nolinestop=True), '1.0') + + self.assertEqual(text.search('test', '1.0', '1.13'), '1.10') + self.assertEqual(text.search('test', '1.0', '1.13', strictlimits=True), '') + self.assertEqual(text.search('test', '1.0', '1.14', strictlimits=True), '1.10') + + var = tkinter.Variable(self.root) + self.assertEqual(text.search('test', '1.0', count=var), '1.10') + self.assertEqual(var.get(), 4 if self.wantobjects else '4') + + # TODO: Add test for elide=True + + def test_search_all(self): + text = self.text + + # pattern and index are obligatory arguments. + self.assertRaises(tkinter.TclError, text.search_all, None, '1.0') + self.assertRaises(tkinter.TclError, text.search_all, 'a', None) + self.assertRaises(tkinter.TclError, text.search_all, None, None) + + # Keyword-only arguments + self.assertRaises(TypeError, text.search_all, 'a', '1.0', 'end', None) + + # Invalid text index. + self.assertRaises(tkinter.TclError, text.search_all, '', 0) + self.assertRaises(tkinter.TclError, text.search_all, '', '') + self.assertRaises(tkinter.TclError, text.search_all, '', 'invalid') + self.assertRaises(tkinter.TclError, text.search_all, '', '1.0', 0) + self.assertRaises(tkinter.TclError, text.search_all, '', '1.0', '') + self.assertRaises(tkinter.TclError, text.search_all, '', '1.0', 'invalid') + + def eq(res, expected): + self.assertIsInstance(res, tuple) + self.assertEqual([str(i) for i in res], expected) + + text.insert('1.0', 'ababa\naba\n64-bit') + + eq(text.search_all('aba', '1.0'), ['1.0', '2.0']) + eq(text.search_all('aba', '1.0', 'end'), ['1.0', '2.0']) + eq(text.search_all('aba', '1.1', 'end'), ['1.2', '2.0']) + eq(text.search_all('aba', '1.1'), ['1.2', '2.0', '1.0']) + + res = text.search_all('', '1.0') # empty pattern + eq(res[:5], ['1.0', '1.1', '1.2', '1.3', '1.4']) + eq(res[-5:], ['3.2', '3.3', '3.4', '3.5', '3.6']) + eq(text.search_all('nonexistent', '1.0'), []) + eq(text.search_all('-bit', '1.0'), ['3.2']) # starts with a hyphen + + eq(text.search_all('aba', '1.0', 'end', forwards=True), ['1.0', '2.0']) + eq(text.search_all('aba', 'end', '1.0', backwards=True), ['2.0', '1.2']) + + eq(text.search_all('aba', '1.0', overlap=True), ['1.0', '1.2', '2.0']) + eq(text.search_all('aba', 'end', '1.0', overlap=True, backwards=True), ['2.0', '1.2', '1.0']) + + eq(text.search_all('aba', '1.0', exact=True), ['1.0', '2.0']) + eq(text.search_all('a.a', '1.0', exact=True), []) + eq(text.search_all('a.a', '1.0', regexp=True), ['1.0', '2.0']) + + eq(text.search_all('ABA', '1.0'), []) + eq(text.search_all('ABA', '1.0', nocase=True), ['1.0', '2.0']) + + eq(text.search_all('a.a', '1.0', regexp=True), ['1.0', '2.0']) + eq(text.search_all('a.a', '1.0', regexp=True, nolinestop=True), ['1.0', '1.4']) + + eq(text.search_all('aba', '1.0', '2.2'), ['1.0', '2.0']) + eq(text.search_all('aba', '1.0', '2.2', strictlimits=True), ['1.0']) + eq(text.search_all('aba', '1.0', '2.3', strictlimits=True), ['1.0', '2.0']) + + var = tkinter.Variable(self.root) + eq(text.search_all('aba', '1.0', count=var), ['1.0', '2.0']) + self.assertEqual(var.get(), (3, 3) if self.wantobjects else '3 3') + + # TODO: Add test for elide=True def test_count(self): text = self.text diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index c54530740395f7..737583a42c6399 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -4049,8 +4049,9 @@ def scan_dragto(self, x, y): self.tk.call(self._w, 'scan', 'dragto', x, y) def search(self, pattern, index, stopindex=None, - forwards=None, backwards=None, exact=None, - regexp=None, nocase=None, count=None, elide=None): + forwards=None, backwards=None, exact=None, + regexp=None, nocase=None, count=None, + elide=None, *, nolinestop=None, strictlimits=None): """Search PATTERN beginning from INDEX until STOPINDEX. Return the index of the first character of a match or an empty string.""" @@ -4062,12 +4063,39 @@ def search(self, pattern, index, stopindex=None, if nocase: args.append('-nocase') if elide: args.append('-elide') if count: args.append('-count'); args.append(count) + if nolinestop: args.append('-nolinestop') + if strictlimits: args.append('-strictlimits') if pattern and pattern[0] == '-': args.append('--') args.append(pattern) args.append(index) - if stopindex: args.append(stopindex) + if stopindex is not None: args.append(stopindex) return str(self.tk.call(tuple(args))) + def search_all(self, pattern, index, stopindex=None, *, + forwards=None, backwards=None, exact=None, + regexp=None, nocase=None, count=None, + elide=None, nolinestop=None, overlap=None, + strictlimits=None): + """Search all occurrences of PATTERN from INDEX to STOPINDEX. + Return a tuple of indices where matches begin.""" + args = [self._w, 'search', '-all'] + if forwards: args.append('-forwards') + if backwards: args.append('-backwards') + if exact: args.append('-exact') + if regexp: args.append('-regexp') + if nocase: args.append('-nocase') + if elide: args.append('-elide') + if count: args.append('-count'); args.append(count) + if nolinestop: args.append('-nolinestop') + if overlap: args.append('-overlap') + if strictlimits: args.append('-strictlimits') + if pattern and pattern[0] == '-': args.append('--') + args.append(pattern) + args.append(index) + if stopindex is not None: args.append(stopindex) + result = self.tk.call(tuple(args)) + return self.tk.splitlist(result) + def see(self, index): """Scroll such that the character at INDEX is visible.""" self.tk.call(self._w, 'see', index) diff --git a/Misc/NEWS.d/next/Library/2025-03-04-17-19-26.gh-issue-130693.Kv01r8.rst b/Misc/NEWS.d/next/Library/2025-03-04-17-19-26.gh-issue-130693.Kv01r8.rst new file mode 100644 index 00000000000000..b175ab7cad468a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-03-04-17-19-26.gh-issue-130693.Kv01r8.rst @@ -0,0 +1 @@ +Add support for ``-nolinestop``, and ``-strictlimits`` options to :meth:`!tkinter.Text.search`. Also add the :meth:`!tkinter.Text.search_all` method for ``-all`` and ``-overlap`` options. From cc6b62ac561e857a2cc4eb4f43e1e0e9f53c09f1 Mon Sep 17 00:00:00 2001 From: Semyon Moroz Date: Mon, 17 Nov 2025 18:51:21 +0400 Subject: [PATCH 078/638] gh-130160: Add anchors to CLI Usage section for `cmdline` (#133182) --- Doc/library/cmdline.rst | 10 +++++----- Doc/library/ensurepip.rst | 4 +++- Doc/library/gzip.rst | 4 ++-- Doc/library/idle.rst | 4 +++- Doc/library/inspect.rst | 2 +- Doc/library/pdb.rst | 6 +++++- Doc/library/site.rst | 2 +- Doc/library/webbrowser.rst | 7 ++++++- 8 files changed, 26 insertions(+), 13 deletions(-) diff --git a/Doc/library/cmdline.rst b/Doc/library/cmdline.rst index 16c67ddbf7cec2..c43b10157f9aea 100644 --- a/Doc/library/cmdline.rst +++ b/Doc/library/cmdline.rst @@ -16,17 +16,17 @@ The following modules have a command-line interface. * :ref:`dis ` * :ref:`doctest ` * :mod:`!encodings.rot_13` -* :mod:`ensurepip` +* :ref:`ensurepip ` * :mod:`filecmp` * :mod:`fileinput` * :mod:`ftplib` * :ref:`gzip ` * :ref:`http.server ` -* :mod:`!idlelib` +* :ref:`idlelib ` * :ref:`inspect ` * :ref:`json ` * :ref:`mimetypes ` -* :mod:`pdb` +* :ref:`pdb ` * :ref:`pickle ` * :ref:`pickletools ` * :ref:`platform ` @@ -52,8 +52,8 @@ The following modules have a command-line interface. * :mod:`turtledemo` * :ref:`unittest ` * :ref:`uuid ` -* :mod:`venv` -* :mod:`webbrowser` +* :ref:`venv ` +* :ref:`webbrowser ` * :ref:`zipapp ` * :ref:`zipfile ` diff --git a/Doc/library/ensurepip.rst b/Doc/library/ensurepip.rst index 165b9a9f823154..32b92c01570004 100644 --- a/Doc/library/ensurepip.rst +++ b/Doc/library/ensurepip.rst @@ -42,7 +42,9 @@ when creating a virtual environment) or after explicitly uninstalling .. include:: ../includes/wasm-mobile-notavail.rst -Command line interface +.. _ensurepip-cli: + +Command-line interface ---------------------- .. program:: ensurepip diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index cb36be42a83f12..d23c0741ddbecd 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -283,7 +283,7 @@ Example of how to GZIP compress a binary string:: .. _gzip-cli: -Command Line Interface +Command-line interface ---------------------- The :mod:`gzip` module provides a simple command line interface to compress or @@ -296,7 +296,7 @@ Once executed the :mod:`gzip` module keeps the input file(s). Add a new command line interface with a usage. By default, when you will execute the CLI, the default compression level is 6. -Command line options +Command-line options ^^^^^^^^^^^^^^^^^^^^ .. option:: file diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index 52e3726a0f5af5..a16f46ef812400 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -661,7 +661,9 @@ looked for in the user's home directory. Statements in this file will be executed in the Tk namespace, so this file is not useful for importing functions to be used from IDLE's Python shell. -Command line usage +.. _idlelib-cli: + +Command-line usage ^^^^^^^^^^^^^^^^^^ .. program:: idle diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 13a352cbdb2cdc..c00db31a8ec051 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -1788,7 +1788,7 @@ Buffer flags .. _inspect-module-cli: -Command Line Interface +Command-line interface ---------------------- The :mod:`inspect` module also provides a basic introspection capability diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index 90dc6648045f27..0bbdc42535290a 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -76,6 +76,10 @@ The debugger's prompt is ``(Pdb)``, which is the indicator that you are in debug .. _pdb-cli: + +Command-line interface +---------------------- + .. program:: pdb You can also invoke :mod:`pdb` from the command line to debug other scripts. For @@ -334,7 +338,7 @@ access further features, you have to do this yourself: .. _debugger-commands: -Debugger Commands +Debugger commands ----------------- The commands recognized by the debugger are listed below. Most commands can be diff --git a/Doc/library/site.rst b/Doc/library/site.rst index e98dd83b60eb60..d93e4dc7c75f1a 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -270,7 +270,7 @@ Module contents .. _site-commandline: -Command Line Interface +Command-line interface ---------------------- .. program:: site diff --git a/Doc/library/webbrowser.rst b/Doc/library/webbrowser.rst index fd6abc70261019..a2103d8fdd8efe 100644 --- a/Doc/library/webbrowser.rst +++ b/Doc/library/webbrowser.rst @@ -49,6 +49,11 @@ a new tab, with the browser being brought to the foreground. The use of the :mod:`webbrowser` module on iOS requires the :mod:`ctypes` module. If :mod:`ctypes` isn't available, calls to :func:`.open` will fail. +.. _webbrowser-cli: + +Command-line interface +---------------------- + .. program:: webbrowser The script :program:`webbrowser` can be used as a command-line interface for the @@ -232,7 +237,7 @@ Here are some simple examples:: .. _browser-controllers: -Browser Controller Objects +Browser controller objects -------------------------- Browser controllers provide the :attr:`~controller.name` attribute, From 274a26cca8e3d2f4de0283d4acbc80be391a5f6a Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Mon, 17 Nov 2025 16:32:08 +0000 Subject: [PATCH 079/638] gh-135953: Simplify GC markers in the tachyon profiler (#141666) --- Lib/profiling/sampling/gecko_collector.py | 15 +++++++-------- Lib/test/test_profiling/test_sampling_profiler.py | 6 ++---- Modules/_remote_debugging_module.c | 11 ----------- 3 files changed, 9 insertions(+), 23 deletions(-) diff --git a/Lib/profiling/sampling/gecko_collector.py b/Lib/profiling/sampling/gecko_collector.py index 6c6700f113083e..21c427b7c862a4 100644 --- a/Lib/profiling/sampling/gecko_collector.py +++ b/Lib/profiling/sampling/gecko_collector.py @@ -141,7 +141,6 @@ def collect(self, stack_frames): for thread_info in interpreter_info.threads: frames = thread_info.frame_info tid = thread_info.thread_id - gc_collecting = thread_info.gc_collecting # Initialize thread if needed if tid not in self.threads: @@ -197,16 +196,16 @@ def collect(self, stack_frames): self._add_marker(tid, "Waiting for GIL", self.gil_wait_start.pop(tid), current_time, CATEGORY_GIL) - # Track GC events - attribute to all threads that hold the GIL during GC - # (GC is interpreter-wide but runs on whichever thread(s) have the GIL) - # If GIL switches during GC, multiple threads will get GC markers - if gc_collecting and has_gil: - # Start GC marker if not already started for this thread + # Track GC events by detecting frames in the stack trace + # This leverages the improved GC frame tracking from commit 336366fd7ca + # which precisely identifies the thread that initiated GC collection + has_gc_frame = any(frame[2] == "" for frame in frames) + if has_gc_frame: + # This thread initiated GC collection if tid not in self.gc_start_per_thread: self.gc_start_per_thread[tid] = current_time elif tid in self.gc_start_per_thread: - # End GC marker if it was running for this thread - # (either GC finished or thread lost GIL) + # End GC marker when no more GC frames are detected self._add_marker(tid, "GC Collecting", self.gc_start_per_thread.pop(tid), current_time, CATEGORY_GC) diff --git a/Lib/test/test_profiling/test_sampling_profiler.py b/Lib/test/test_profiling/test_sampling_profiler.py index a24dbb55cd7bab..2d00173c22c419 100644 --- a/Lib/test/test_profiling/test_sampling_profiler.py +++ b/Lib/test/test_profiling/test_sampling_profiler.py @@ -63,14 +63,13 @@ def __repr__(self): class MockThreadInfo: """Mock ThreadInfo for testing since the real one isn't accessible.""" - def __init__(self, thread_id, frame_info, status=0, gc_collecting=False): # Default to THREAD_STATE_RUNNING (0) + def __init__(self, thread_id, frame_info, status=0): # Default to THREAD_STATE_RUNNING (0) self.thread_id = thread_id self.frame_info = frame_info self.status = status - self.gc_collecting = gc_collecting def __repr__(self): - return f"MockThreadInfo(thread_id={self.thread_id}, frame_info={self.frame_info}, status={self.status}, gc_collecting={self.gc_collecting})" + return f"MockThreadInfo(thread_id={self.thread_id}, frame_info={self.frame_info}, status={self.status})" class MockInterpreterInfo: @@ -2742,7 +2741,6 @@ def __init__(self, thread_id, frame_info, status): self.thread_id = thread_id self.frame_info = frame_info self.status = status - self.gc_collecting = False # Create test data: active thread (HAS_GIL | ON_CPU), idle thread (neither), and another active thread ACTIVE_STATUS = THREAD_STATUS_HAS_GIL | THREAD_STATUS_ON_CPU # Has GIL and on CPU diff --git a/Modules/_remote_debugging_module.c b/Modules/_remote_debugging_module.c index 51b3c6bac02b54..6544e3a0ce6876 100644 --- a/Modules/_remote_debugging_module.c +++ b/Modules/_remote_debugging_module.c @@ -186,7 +186,6 @@ static PyStructSequence_Field ThreadInfo_fields[] = { {"thread_id", "Thread ID"}, {"status", "Thread status (flags: HAS_GIL, ON_CPU, UNKNOWN or legacy enum)"}, {"frame_info", "Frame information"}, - {"gc_collecting", "Whether GC is collecting (interpreter-level)"}, {NULL} }; @@ -2726,8 +2725,6 @@ unwind_stack_for_thread( goto error; } - int gc_collecting = GET_MEMBER(int, gc_state, unwinder->debug_offsets.gc.collecting); - // Calculate thread status using flags (always) int status_flags = 0; @@ -2827,18 +2824,10 @@ unwind_stack_for_thread( goto error; } - PyObject *py_gc_collecting = PyBool_FromLong(gc_collecting); - if (py_gc_collecting == NULL) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create gc_collecting"); - Py_DECREF(py_status); - goto error; - } - // py_status contains status flags (bitfield) PyStructSequence_SetItem(result, 0, thread_id); PyStructSequence_SetItem(result, 1, py_status); // Steals reference PyStructSequence_SetItem(result, 2, frame_info); // Steals reference - PyStructSequence_SetItem(result, 3, py_gc_collecting); // Steals reference cleanup_stack_chunks(&chunks); return result; From 6b1bdf6c7a6c87f12a247a125e25f8e721cc731e Mon Sep 17 00:00:00 2001 From: Krishna Chaitanya <141550576+XChaitanyaX@users.noreply.github.com> Date: Mon, 17 Nov 2025 22:59:06 +0530 Subject: [PATCH 080/638] gh-141497: Make ipaddress.IP{v4,v6}Network.hosts() always returning an iterator (GH-141547) --- Lib/ipaddress.py | 4 +-- Lib/test/test_ipaddress.py | 34 +++++++++++++++++++ ...-11-14-16-24-20.gh-issue-141497.L_CxDJ.rst | 4 +++ 3 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-14-16-24-20.gh-issue-141497.L_CxDJ.rst diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index aa0cf4a0620cd0..f1062a8cd052a5 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -1546,7 +1546,7 @@ def __init__(self, address, strict=True): if self._prefixlen == (self.max_prefixlen - 1): self.hosts = self.__iter__ elif self._prefixlen == (self.max_prefixlen): - self.hosts = lambda: [IPv4Address(addr)] + self.hosts = lambda: iter((IPv4Address(addr),)) @property @functools.lru_cache() @@ -2337,7 +2337,7 @@ def __init__(self, address, strict=True): if self._prefixlen == (self.max_prefixlen - 1): self.hosts = self.__iter__ elif self._prefixlen == self.max_prefixlen: - self.hosts = lambda: [IPv6Address(addr)] + self.hosts = lambda: iter((IPv6Address(addr),)) def hosts(self): """Generate Iterator over usable hosts in a network. diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index 11721a59972672..3f017b97dc28a3 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -12,6 +12,7 @@ import pickle import ipaddress import weakref +from collections.abc import Iterator from test.support import LARGEST, SMALLEST @@ -1472,18 +1473,27 @@ def testGetSupernet4(self): self.ipv6_scoped_network.supernet(new_prefix=62)) def testHosts(self): + hosts = self.ipv4_network.hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(ipaddress.IPv4Address('1.2.3.1'), next(hosts)) hosts = list(self.ipv4_network.hosts()) self.assertEqual(254, len(hosts)) self.assertEqual(ipaddress.IPv4Address('1.2.3.1'), hosts[0]) self.assertEqual(ipaddress.IPv4Address('1.2.3.254'), hosts[-1]) ipv6_network = ipaddress.IPv6Network('2001:658:22a:cafe::/120') + hosts = ipv6_network.hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(ipaddress.IPv6Address('2001:658:22a:cafe::1'), next(hosts)) hosts = list(ipv6_network.hosts()) self.assertEqual(255, len(hosts)) self.assertEqual(ipaddress.IPv6Address('2001:658:22a:cafe::1'), hosts[0]) self.assertEqual(ipaddress.IPv6Address('2001:658:22a:cafe::ff'), hosts[-1]) ipv6_scoped_network = ipaddress.IPv6Network('2001:658:22a:cafe::%scope/120') + hosts = ipv6_scoped_network.hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual((ipaddress.IPv6Address('2001:658:22a:cafe::1')), next(hosts)) hosts = list(ipv6_scoped_network.hosts()) self.assertEqual(255, len(hosts)) self.assertEqual(ipaddress.IPv6Address('2001:658:22a:cafe::1'), hosts[0]) @@ -1494,6 +1504,12 @@ def testHosts(self): ipaddress.IPv4Address('2.0.0.1')] str_args = '2.0.0.0/31' tpl_args = ('2.0.0.0', 31) + hosts = ipaddress.ip_network(str_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) + hosts = ipaddress.ip_network(tpl_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) self.assertEqual(addrs, list(ipaddress.ip_network(str_args).hosts())) self.assertEqual(addrs, list(ipaddress.ip_network(tpl_args).hosts())) self.assertEqual(list(ipaddress.ip_network(str_args).hosts()), @@ -1503,6 +1519,12 @@ def testHosts(self): addrs = [ipaddress.IPv4Address('1.2.3.4')] str_args = '1.2.3.4/32' tpl_args = ('1.2.3.4', 32) + hosts = ipaddress.ip_network(str_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) + hosts = ipaddress.ip_network(tpl_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) self.assertEqual(addrs, list(ipaddress.ip_network(str_args).hosts())) self.assertEqual(addrs, list(ipaddress.ip_network(tpl_args).hosts())) self.assertEqual(list(ipaddress.ip_network(str_args).hosts()), @@ -1512,6 +1534,12 @@ def testHosts(self): ipaddress.IPv6Address('2001:658:22a:cafe::1')] str_args = '2001:658:22a:cafe::/127' tpl_args = ('2001:658:22a:cafe::', 127) + hosts = ipaddress.ip_network(str_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) + hosts = ipaddress.ip_network(tpl_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) self.assertEqual(addrs, list(ipaddress.ip_network(str_args).hosts())) self.assertEqual(addrs, list(ipaddress.ip_network(tpl_args).hosts())) self.assertEqual(list(ipaddress.ip_network(str_args).hosts()), @@ -1520,6 +1548,12 @@ def testHosts(self): addrs = [ipaddress.IPv6Address('2001:658:22a:cafe::1'), ] str_args = '2001:658:22a:cafe::1/128' tpl_args = ('2001:658:22a:cafe::1', 128) + hosts = ipaddress.ip_network(str_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) + hosts = ipaddress.ip_network(tpl_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) self.assertEqual(addrs, list(ipaddress.ip_network(str_args).hosts())) self.assertEqual(addrs, list(ipaddress.ip_network(tpl_args).hosts())) self.assertEqual(list(ipaddress.ip_network(str_args).hosts()), diff --git a/Misc/NEWS.d/next/Library/2025-11-14-16-24-20.gh-issue-141497.L_CxDJ.rst b/Misc/NEWS.d/next/Library/2025-11-14-16-24-20.gh-issue-141497.L_CxDJ.rst new file mode 100644 index 00000000000000..328bfe067ad96b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-14-16-24-20.gh-issue-141497.L_CxDJ.rst @@ -0,0 +1,4 @@ +:mod:`ipaddress`: ensure that the methods +:meth:`IPv4Network.hosts() ` and +:meth:`IPv6Network.hosts() ` always return an +iterator. From 5d2eb98a91f2cd703d14f38c751ac7f52b2d7148 Mon Sep 17 00:00:00 2001 From: Louis Date: Mon, 17 Nov 2025 18:47:00 +0100 Subject: [PATCH 081/638] gh-140578: Delete unnecessary NEWS entry (#141427) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- .../2025-10-27-23-06-01.gh-issue-140578.FMBdEn.rst | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 Misc/NEWS.d/next/Documentation/2025-10-27-23-06-01.gh-issue-140578.FMBdEn.rst diff --git a/Misc/NEWS.d/next/Documentation/2025-10-27-23-06-01.gh-issue-140578.FMBdEn.rst b/Misc/NEWS.d/next/Documentation/2025-10-27-23-06-01.gh-issue-140578.FMBdEn.rst deleted file mode 100644 index 702d38d4d24df6..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2025-10-27-23-06-01.gh-issue-140578.FMBdEn.rst +++ /dev/null @@ -1,3 +0,0 @@ -Remove outdated sencence in the documentation for :mod:`multiprocessing`, -that implied that :class:`concurrent.futures.ThreadPoolExecutor` did not -exist. From b3626321b6ebb46dd24acee2aa806450e70febfc Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Mon, 17 Nov 2025 14:40:47 -0500 Subject: [PATCH 082/638] gh-141004: Document `PyODict*` APIs (GH-141136) --- Doc/c-api/dict.rst | 89 ++++++++++++++++++++++++++++++++++++++++++ Doc/c-api/iterator.rst | 1 + 2 files changed, 90 insertions(+) diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index b7f201811aad6c..ede1699cfeb653 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -477,3 +477,92 @@ Dictionary View Objects Return true if *op* is an instance of a dictionary items view. This function always succeeds. + + +Ordered Dictionaries +^^^^^^^^^^^^^^^^^^^^ + +Python's C API provides interface for :class:`collections.OrderedDict` from C. +Since Python 3.7, dictionaries are ordered by default, so there is usually +little need for these functions; prefer ``PyDict*`` where possible. + + +.. c:var:: PyTypeObject PyODict_Type + + Type object for ordered dictionaries. This is the same object as + :class:`collections.OrderedDict` in the Python layer. + + +.. c:function:: int PyODict_Check(PyObject *od) + + Return true if *od* is an ordered dictionary object or an instance of a + subtype of the :class:`~collections.OrderedDict` type. This function + always succeeds. + + +.. c:function:: int PyODict_CheckExact(PyObject *od) + + Return true if *od* is an ordered dictionary object, but not an instance of + a subtype of the :class:`~collections.OrderedDict` type. + This function always succeeds. + + +.. c:var:: PyTypeObject PyODictKeys_Type + + Analogous to :c:type:`PyDictKeys_Type` for ordered dictionaries. + + +.. c:var:: PyTypeObject PyODictValues_Type + + Analogous to :c:type:`PyDictValues_Type` for ordered dictionaries. + + +.. c:var:: PyTypeObject PyODictItems_Type + + Analogous to :c:type:`PyDictItems_Type` for ordered dictionaries. + + +.. c:function:: PyObject *PyODict_New(void) + + Return a new empty ordered dictionary, or ``NULL`` on failure. + + This is analogous to :c:func:`PyDict_New`. + + +.. c:function:: int PyODict_SetItem(PyObject *od, PyObject *key, PyObject *value) + + Insert *value* into the ordered dictionary *od* with a key of *key*. + Return ``0`` on success or ``-1`` with an exception set on failure. + + This is analogous to :c:func:`PyDict_SetItem`. + + +.. c:function:: int PyODict_DelItem(PyObject *od, PyObject *key) + + Remove the entry in the ordered dictionary *od* with key *key*. + Return ``0`` on success or ``-1`` with an exception set on failure. + + This is analogous to :c:func:`PyDict_DelItem`. + + +These are :term:`soft deprecated` aliases to ``PyDict`` APIs: + + +.. list-table:: + :widths: auto + :header-rows: 1 + + * * ``PyODict`` + * ``PyDict`` + * * .. c:macro:: PyODict_GetItem(od, key) + * :c:func:`PyDict_GetItem` + * * .. c:macro:: PyODict_GetItemWithError(od, key) + * :c:func:`PyDict_GetItemWithError` + * * .. c:macro:: PyODict_GetItemString(od, key) + * :c:func:`PyDict_GetItemString` + * * .. c:macro:: PyODict_Contains(od, key) + * :c:func:`PyDict_Contains` + * * .. c:macro:: PyODict_Size(od) + * :c:func:`PyDict_Size` + * * .. c:macro:: PyODict_SIZE(od) + * :c:func:`PyDict_GET_SIZE` diff --git a/Doc/c-api/iterator.rst b/Doc/c-api/iterator.rst index 7eaf72ec55fd77..bfbfe3c9279980 100644 --- a/Doc/c-api/iterator.rst +++ b/Doc/c-api/iterator.rst @@ -108,6 +108,7 @@ Other Iterator Objects .. c:var:: PyTypeObject PyDictRevIterValue_Type .. c:var:: PyTypeObject PyDictIterItem_Type .. c:var:: PyTypeObject PyDictRevIterItem_Type +.. c:var:: PyTypeObject PyODictIter_Type Type objects for iterators of various built-in objects. From 16ea9505ce690485bab38691e5a83f467757fc03 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Mon, 17 Nov 2025 22:52:13 +0000 Subject: [PATCH 083/638] gh-141004: Document `Py_MEMCPY` (GH-141676) --- Doc/c-api/intro.rst | 8 ++++++++ Misc/NEWS.d/3.14.0a1.rst | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index c76cc2f70ecccf..bace21b7981091 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -183,6 +183,14 @@ complete listing. .. versionadded:: 3.6 +.. c:macro:: Py_MEMCPY(dest, src, n) + + This is a :term:`soft deprecated` alias to :c:func:`!memcpy`. + Use :c:func:`!memcpy` directly instead. + + .. deprecated:: 3.14 + The macro is :term:`soft deprecated`. + .. c:macro:: Py_MIN(x, y) Return the minimum value between ``x`` and ``y``. diff --git a/Misc/NEWS.d/3.14.0a1.rst b/Misc/NEWS.d/3.14.0a1.rst index 305a0b65b98e6a..1938976fa4226a 100644 --- a/Misc/NEWS.d/3.14.0a1.rst +++ b/Misc/NEWS.d/3.14.0a1.rst @@ -6092,7 +6092,7 @@ Patch by Victor Stinner. .. nonce: qOr9GF .. section: C API -Soft deprecate the :c:macro:`!Py_MEMCPY` macro: use directly ``memcpy()`` +Soft deprecate the :c:macro:`Py_MEMCPY` macro: use directly ``memcpy()`` instead. Patch by Victor Stinner. .. From 4867f717e21c3b5f0ad0e81f950c69dac6c95e6e Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Tue, 18 Nov 2025 02:26:40 +0000 Subject: [PATCH 084/638] gh-140729: Fix subprocess handling in test_process_pool_executor_pickle (#141688) --- Lib/test/test_profiling/test_sampling_profiler.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_profiling/test_sampling_profiler.py b/Lib/test/test_profiling/test_sampling_profiler.py index 2d00173c22c419..c2cc2ddd48a02c 100644 --- a/Lib/test/test_profiling/test_sampling_profiler.py +++ b/Lib/test/test_profiling/test_sampling_profiler.py @@ -3311,6 +3311,8 @@ def test_native_frames_disabled(self): self.assertNotIn("", output) +@requires_subprocess() +@skip_if_not_supported class TestProcessPoolExecutorSupport(unittest.TestCase): """ Test that ProcessPoolExecutor works correctly with profiling.sampling. @@ -3339,12 +3341,15 @@ def worker(x): "-d", "5", "-i", "100000", script, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True ) as proc: - proc.wait(timeout=SHORT_TIMEOUT) - stdout = proc.stdout.read() - stderr = proc.stderr.read() + try: + stdout, stderr = proc.communicate(timeout=SHORT_TIMEOUT) + except subprocess.TimeoutExpired: + proc.kill() + stdout, stderr = proc.communicate() if "PermissionError" in stderr: self.skipTest("Insufficient permissions for remote profiling") From 58f3fe0d9b9882656e629e8caab687c7fcb21b36 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 18 Nov 2025 01:10:32 -0800 Subject: [PATCH 085/638] gh-129005: Remove copies from _pyio using take_bytes (#141539) Memory usage now matches that of _io for large files. --- Lib/_pyio.py | 8 ++++---- Lib/test/test_io/test_bufferedio.py | 3 ++- Lib/test/test_io/test_largefile.py | 6 ++---- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 423178e87a8684..69a088df8fc987 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -546,7 +546,7 @@ def nreadahead(): res += b if res.endswith(b"\n"): break - return bytes(res) + return res.take_bytes() def __iter__(self): self._checkClosed() @@ -620,7 +620,7 @@ def read(self, size=-1): if n < 0 or n > len(b): raise ValueError(f"readinto returned {n} outside buffer size {len(b)}") del b[n:] - return bytes(b) + return b.take_bytes() def readall(self): """Read until EOF, using multiple read() call.""" @@ -628,7 +628,7 @@ def readall(self): while data := self.read(DEFAULT_BUFFER_SIZE): res += data if res: - return bytes(res) + return res.take_bytes() else: # b'' or None return data @@ -1738,7 +1738,7 @@ def readall(self): assert len(result) - bytes_read >= 1, \ "os.readinto buffer size 0 will result in erroneous EOF / returns 0" result.resize(bytes_read) - return bytes(result) + return result.take_bytes() def readinto(self, buffer): """Same as RawIOBase.readinto().""" diff --git a/Lib/test/test_io/test_bufferedio.py b/Lib/test/test_io/test_bufferedio.py index 30c34e818b1572..3278665bdc9dd3 100644 --- a/Lib/test/test_io/test_bufferedio.py +++ b/Lib/test/test_io/test_bufferedio.py @@ -1277,7 +1277,8 @@ def test_flush_and_readinto(self): def _readinto(bufio, n=-1): b = bytearray(n if n >= 0 else 9999) n = bufio.readinto(b) - return bytes(b[:n]) + b.resize(n) + return b.take_bytes() self.check_flush_and_read(_readinto) def test_flush_and_peek(self): diff --git a/Lib/test/test_io/test_largefile.py b/Lib/test/test_io/test_largefile.py index 41f7b70e5cfe81..438a90a92ed588 100644 --- a/Lib/test/test_io/test_largefile.py +++ b/Lib/test/test_io/test_largefile.py @@ -56,9 +56,7 @@ class TestFileMethods(LargeFileTest): (i.e. > 2 GiB) files. """ - # _pyio.FileIO.readall() uses a temporary bytearray then casted to bytes, - # so memuse=2 is needed - @bigmemtest(size=size, memuse=2, dry_run=False) + @bigmemtest(size=size, memuse=1, dry_run=False) def test_large_read(self, _size): # bpo-24658: Test that a read greater than 2GB does not fail. with self.open(TESTFN, "rb") as f: @@ -154,7 +152,7 @@ def test_seekable(self): f.seek(pos) self.assertTrue(f.seekable()) - @bigmemtest(size=size, memuse=2, dry_run=False) + @bigmemtest(size=size, memuse=1, dry_run=False) def test_seek_readall(self, _size): # Seek which doesn't change position should readall successfully. with self.open(TESTFN, 'rb') as f: From 630cd37bfae0fc4021d9e9461b94d36e7ce6b95c Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Tue, 18 Nov 2025 12:17:37 +0300 Subject: [PATCH 086/638] gh-141004: Document Py_HUGE_VAL/IS_FINITE/IS_INFINITE/IS_NAN (#141544) Co-authored-by: Victor Stinner Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/c-api/float.rst | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index 79de5daaa90d8f..b0d440580b9886 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -87,7 +87,7 @@ Floating-Point Objects ```` header. .. deprecated:: 3.15 - The macro is soft deprecated. + The macro is :term:`soft deprecated`. .. c:macro:: Py_NAN @@ -99,6 +99,14 @@ Floating-Point Objects the C11 standard ```` header. +.. c:macro:: Py_HUGE_VAL + + Equivalent to :c:macro:`!INFINITY`. + + .. deprecated:: 3.14 + The macro is :term:`soft deprecated`. + + .. c:macro:: Py_MATH_E The definition (accurate for a :c:expr:`double` type) of the :data:`math.e` constant. @@ -147,6 +155,34 @@ Floating-Point Objects return PyFloat_FromDouble(copysign(INFINITY, sign)); +.. c:macro:: Py_IS_FINITE(X) + + Return ``1`` if the given floating-point number *X* is finite, + that is, it is normal, subnormal or zero, but not infinite or NaN. + Return ``0`` otherwise. + + .. deprecated:: 3.14 + The macro is :term:`soft deprecated`. Use :c:macro:`!isfinite` instead. + + +.. c:macro:: Py_IS_INFINITY(X) + + Return ``1`` if the given floating-point number *X* is positive or negative + infinity. Return ``0`` otherwise. + + .. deprecated:: 3.14 + The macro is :term:`soft deprecated`. Use :c:macro:`!isinf` instead. + + +.. c:macro:: Py_IS_NAN(X) + + Return ``1`` if the given floating-point number *X* is a not-a-number (NaN) + value. Return ``0`` otherwise. + + .. deprecated:: 3.14 + The macro is :term:`soft deprecated`. Use :c:macro:`!isnan` instead. + + Pack and Unpack functions ------------------------- From b87613f21474ea848fec435cbfe63d8cb1c7c44c Mon Sep 17 00:00:00 2001 From: Mariusz Felisiak Date: Tue, 18 Nov 2025 11:32:15 +0100 Subject: [PATCH 087/638] Add missing backticks in os and decimal docs (#141699) --- Doc/library/decimal.rst | 2 +- Doc/library/os.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst index 985153b5443f5c..ba882f10bbe2b8 100644 --- a/Doc/library/decimal.rst +++ b/Doc/library/decimal.rst @@ -264,7 +264,7 @@ allows the settings to be changed. This approach meets the needs of most applications. For more advanced work, it may be useful to create alternate contexts using the -Context() constructor. To make an alternate active, use the :func:`setcontext` +:meth:`Context` constructor. To make an alternate active, use the :func:`setcontext` function. In accordance with the standard, the :mod:`decimal` module provides two ready to diff --git a/Doc/library/os.rst b/Doc/library/os.rst index dbc3c92c8798b5..7dc6c177268ec2 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -558,7 +558,7 @@ process and user. .. function:: initgroups(username, gid, /) - Call the system initgroups() to initialize the group access list with all of + Call the system ``initgroups()`` to initialize the group access list with all of the groups of which the specified username is a member, plus the specified group id. From b420f6be53efdf40f552c94f19a7ce85f882b5e2 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 18 Nov 2025 13:31:48 +0000 Subject: [PATCH 088/638] GH-139109: Support switch/case dispatch with the tracing interpreter. (GH-141703) --- .github/workflows/jit.yml | 26 +- Include/internal/pycore_magic_number.h | 3 +- Include/internal/pycore_opcode_metadata.h | 9 +- Include/internal/pycore_optimizer.h | 2 +- Include/internal/pycore_uop_ids.h | 1 + Include/opcode_ids.h | 47 +- Lib/_opcode_metadata.py | 47 +- Python/bytecodes.c | 9 +- Python/ceval.c | 4 + Python/ceval_macros.h | 10 +- Python/executor_cases.c.h | 2 + Python/generated_cases.c.h | 113 +-- Python/instrumentation.c | 4 +- Python/opcode_targets.h | 910 +++++++++++----------- Python/optimizer_cases.c.h | 2 + Tools/cases_generator/analyzer.py | 7 +- Tools/cases_generator/target_generator.py | 4 +- Tools/cases_generator/tier1_generator.py | 2 +- 18 files changed, 617 insertions(+), 585 deletions(-) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 3349eb042425dd..62325250bd368e 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -57,10 +57,9 @@ jobs: fail-fast: false matrix: target: -# To re-enable later when we support these. -# - i686-pc-windows-msvc/msvc -# - x86_64-pc-windows-msvc/msvc -# - aarch64-pc-windows-msvc/msvc + - i686-pc-windows-msvc/msvc + - x86_64-pc-windows-msvc/msvc + - aarch64-pc-windows-msvc/msvc - x86_64-apple-darwin/clang - aarch64-apple-darwin/clang - x86_64-unknown-linux-gnu/gcc @@ -71,16 +70,15 @@ jobs: llvm: - 21 include: -# To re-enable later when we support these. -# - target: i686-pc-windows-msvc/msvc -# architecture: Win32 -# runner: windows-2022 -# - target: x86_64-pc-windows-msvc/msvc -# architecture: x64 -# runner: windows-2022 -# - target: aarch64-pc-windows-msvc/msvc -# architecture: ARM64 -# runner: windows-11-arm + - target: i686-pc-windows-msvc/msvc + architecture: Win32 + runner: windows-2022 + - target: x86_64-pc-windows-msvc/msvc + architecture: x64 + runner: windows-2022 + - target: aarch64-pc-windows-msvc/msvc + architecture: ARM64 + runner: windows-11-arm - target: x86_64-apple-darwin/clang architecture: x86_64 runner: macos-15-intel diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 7ec7bd1c695516..2fb46a6df50bb3 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -286,6 +286,7 @@ Known values: Python 3.15a1 3653 (Fix handling of opcodes that may leave operands on the stack when optimizing LOAD_FAST) Python 3.15a1 3654 (Fix missing exception handlers in logical expression) Python 3.15a1 3655 (Fix miscompilation of some module-level annotations) + Python 3.15a1 3656 (Add TRACE_RECORD instruction, for platforms with switch based interpreter) Python 3.16 will start with 3700 @@ -299,7 +300,7 @@ PC/launcher.c must also be updated. */ -#define PYC_MAGIC_NUMBER 3655 +#define PYC_MAGIC_NUMBER 3656 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 548627dc7982ec..cca88818c575df 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -488,6 +488,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 1; case TO_BOOL_STR: return 1; + case TRACE_RECORD: + return 0; case UNARY_INVERT: return 1; case UNARY_NEGATIVE: @@ -971,6 +973,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case TO_BOOL_STR: return 1; + case TRACE_RECORD: + return 0; case UNARY_INVERT: return 1; case UNARY_NEGATIVE: @@ -1287,6 +1291,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [TO_BOOL_LIST] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [TO_BOOL_NONE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, [TO_BOOL_STR] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [TRACE_RECORD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_INVERT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNARY_NOT] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, @@ -1738,6 +1743,7 @@ const char *_PyOpcode_OpName[267] = { [TO_BOOL_LIST] = "TO_BOOL_LIST", [TO_BOOL_NONE] = "TO_BOOL_NONE", [TO_BOOL_STR] = "TO_BOOL_STR", + [TRACE_RECORD] = "TRACE_RECORD", [UNARY_INVERT] = "UNARY_INVERT", [UNARY_NEGATIVE] = "UNARY_NEGATIVE", [UNARY_NOT] = "UNARY_NOT", @@ -1809,7 +1815,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [230] = 230, [231] = 231, [232] = 232, - [233] = 233, [BINARY_OP] = BINARY_OP, [BINARY_OP_ADD_FLOAT] = BINARY_OP, [BINARY_OP_ADD_INT] = BINARY_OP, @@ -2025,6 +2030,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [TO_BOOL_LIST] = TO_BOOL, [TO_BOOL_NONE] = TO_BOOL, [TO_BOOL_STR] = TO_BOOL, + [TRACE_RECORD] = TRACE_RECORD, [UNARY_INVERT] = UNARY_INVERT, [UNARY_NEGATIVE] = UNARY_NEGATIVE, [UNARY_NOT] = UNARY_NOT, @@ -2070,7 +2076,6 @@ const uint8_t _PyOpcode_Deopt[256] = { case 230: \ case 231: \ case 232: \ - case 233: \ ; struct pseudo_targets { uint8_t as_sequence; diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 0307a174e77346..e7177552cf666e 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -364,7 +364,7 @@ extern void _Py_ClearExecutorDeletionList(PyInterpreterState *interp); int _PyJit_translate_single_bytecode_to_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, int stop_tracing_opcode); -int +PyAPI_FUNC(int) _PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *curr_instr, _Py_CODEUNIT *start_instr, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 7a33a5b84fd21a..c38f28f9db161b 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -352,6 +352,7 @@ extern "C" { #define _TO_BOOL_LIST 550 #define _TO_BOOL_NONE TO_BOOL_NONE #define _TO_BOOL_STR 551 +#define _TRACE_RECORD TRACE_RECORD #define _UNARY_INVERT UNARY_INVERT #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 1d5c74adefcd35..0d066c169019a7 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -213,28 +213,29 @@ extern "C" { #define UNPACK_SEQUENCE_LIST 207 #define UNPACK_SEQUENCE_TUPLE 208 #define UNPACK_SEQUENCE_TWO_TUPLE 209 -#define INSTRUMENTED_END_FOR 234 -#define INSTRUMENTED_POP_ITER 235 -#define INSTRUMENTED_END_SEND 236 -#define INSTRUMENTED_FOR_ITER 237 -#define INSTRUMENTED_INSTRUCTION 238 -#define INSTRUMENTED_JUMP_FORWARD 239 -#define INSTRUMENTED_NOT_TAKEN 240 -#define INSTRUMENTED_POP_JUMP_IF_TRUE 241 -#define INSTRUMENTED_POP_JUMP_IF_FALSE 242 -#define INSTRUMENTED_POP_JUMP_IF_NONE 243 -#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 244 -#define INSTRUMENTED_RESUME 245 -#define INSTRUMENTED_RETURN_VALUE 246 -#define INSTRUMENTED_YIELD_VALUE 247 -#define INSTRUMENTED_END_ASYNC_FOR 248 -#define INSTRUMENTED_LOAD_SUPER_ATTR 249 -#define INSTRUMENTED_CALL 250 -#define INSTRUMENTED_CALL_KW 251 -#define INSTRUMENTED_CALL_FUNCTION_EX 252 -#define INSTRUMENTED_JUMP_BACKWARD 253 -#define INSTRUMENTED_LINE 254 -#define ENTER_EXECUTOR 255 +#define INSTRUMENTED_END_FOR 233 +#define INSTRUMENTED_POP_ITER 234 +#define INSTRUMENTED_END_SEND 235 +#define INSTRUMENTED_FOR_ITER 236 +#define INSTRUMENTED_INSTRUCTION 237 +#define INSTRUMENTED_JUMP_FORWARD 238 +#define INSTRUMENTED_NOT_TAKEN 239 +#define INSTRUMENTED_POP_JUMP_IF_TRUE 240 +#define INSTRUMENTED_POP_JUMP_IF_FALSE 241 +#define INSTRUMENTED_POP_JUMP_IF_NONE 242 +#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 243 +#define INSTRUMENTED_RESUME 244 +#define INSTRUMENTED_RETURN_VALUE 245 +#define INSTRUMENTED_YIELD_VALUE 246 +#define INSTRUMENTED_END_ASYNC_FOR 247 +#define INSTRUMENTED_LOAD_SUPER_ATTR 248 +#define INSTRUMENTED_CALL 249 +#define INSTRUMENTED_CALL_KW 250 +#define INSTRUMENTED_CALL_FUNCTION_EX 251 +#define INSTRUMENTED_JUMP_BACKWARD 252 +#define INSTRUMENTED_LINE 253 +#define ENTER_EXECUTOR 254 +#define TRACE_RECORD 255 #define ANNOTATIONS_PLACEHOLDER 256 #define JUMP 257 #define JUMP_IF_FALSE 258 @@ -249,7 +250,7 @@ extern "C" { #define HAVE_ARGUMENT 43 #define MIN_SPECIALIZED_OPCODE 129 -#define MIN_INSTRUMENTED_OPCODE 234 +#define MIN_INSTRUMENTED_OPCODE 233 #ifdef __cplusplus } diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index f168d169a32948..e681cb17e43e04 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -208,8 +208,9 @@ 'CACHE': 0, 'RESERVED': 17, 'RESUME': 128, - 'INSTRUMENTED_LINE': 254, - 'ENTER_EXECUTOR': 255, + 'INSTRUMENTED_LINE': 253, + 'ENTER_EXECUTOR': 254, + 'TRACE_RECORD': 255, 'BINARY_SLICE': 1, 'BUILD_TEMPLATE': 2, 'CALL_FUNCTION_EX': 4, @@ -328,26 +329,26 @@ 'UNPACK_EX': 118, 'UNPACK_SEQUENCE': 119, 'YIELD_VALUE': 120, - 'INSTRUMENTED_END_FOR': 234, - 'INSTRUMENTED_POP_ITER': 235, - 'INSTRUMENTED_END_SEND': 236, - 'INSTRUMENTED_FOR_ITER': 237, - 'INSTRUMENTED_INSTRUCTION': 238, - 'INSTRUMENTED_JUMP_FORWARD': 239, - 'INSTRUMENTED_NOT_TAKEN': 240, - 'INSTRUMENTED_POP_JUMP_IF_TRUE': 241, - 'INSTRUMENTED_POP_JUMP_IF_FALSE': 242, - 'INSTRUMENTED_POP_JUMP_IF_NONE': 243, - 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 244, - 'INSTRUMENTED_RESUME': 245, - 'INSTRUMENTED_RETURN_VALUE': 246, - 'INSTRUMENTED_YIELD_VALUE': 247, - 'INSTRUMENTED_END_ASYNC_FOR': 248, - 'INSTRUMENTED_LOAD_SUPER_ATTR': 249, - 'INSTRUMENTED_CALL': 250, - 'INSTRUMENTED_CALL_KW': 251, - 'INSTRUMENTED_CALL_FUNCTION_EX': 252, - 'INSTRUMENTED_JUMP_BACKWARD': 253, + 'INSTRUMENTED_END_FOR': 233, + 'INSTRUMENTED_POP_ITER': 234, + 'INSTRUMENTED_END_SEND': 235, + 'INSTRUMENTED_FOR_ITER': 236, + 'INSTRUMENTED_INSTRUCTION': 237, + 'INSTRUMENTED_JUMP_FORWARD': 238, + 'INSTRUMENTED_NOT_TAKEN': 239, + 'INSTRUMENTED_POP_JUMP_IF_TRUE': 240, + 'INSTRUMENTED_POP_JUMP_IF_FALSE': 241, + 'INSTRUMENTED_POP_JUMP_IF_NONE': 242, + 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 243, + 'INSTRUMENTED_RESUME': 244, + 'INSTRUMENTED_RETURN_VALUE': 245, + 'INSTRUMENTED_YIELD_VALUE': 246, + 'INSTRUMENTED_END_ASYNC_FOR': 247, + 'INSTRUMENTED_LOAD_SUPER_ATTR': 248, + 'INSTRUMENTED_CALL': 249, + 'INSTRUMENTED_CALL_KW': 250, + 'INSTRUMENTED_CALL_FUNCTION_EX': 251, + 'INSTRUMENTED_JUMP_BACKWARD': 252, 'ANNOTATIONS_PLACEHOLDER': 256, 'JUMP': 257, 'JUMP_IF_FALSE': 258, @@ -362,4 +363,4 @@ } HAVE_ARGUMENT = 43 -MIN_INSTRUMENTED_OPCODE = 234 +MIN_INSTRUMENTED_OPCODE = 233 diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 565eaa7a599175..12ee506e4f2bc4 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5636,10 +5636,12 @@ dummy_func( DISPATCH(); } - label(record_previous_inst) { + inst(TRACE_RECORD, (--)) { #if _Py_TIER2 assert(IS_JIT_TRACING()); - int opcode = next_instr->op.code; + next_instr = this_instr; + frame->instr_ptr = prev_instr; + opcode = next_instr->op.code; bool stop_tracing = (opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO || opcode == INTERPRETER_EXIT); @@ -5675,7 +5677,8 @@ dummy_func( } DISPATCH_GOTO_NON_TRACING(); #else - Py_FatalError("JIT label executed in non-jit build."); + (void)prev_instr; + Py_FatalError("JIT instruction executed in non-jit build."); #endif } diff --git a/Python/ceval.c b/Python/ceval.c index 25294ebd993f6c..14fef42ea967d6 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1179,6 +1179,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int uint8_t opcode; /* Current opcode */ int oparg; /* Current opcode argument, if any */ assert(tstate->current_frame == NULL || tstate->current_frame->stackpointer != NULL); +#if !USE_COMPUTED_GOTOS + uint8_t tracing_mode = 0; + uint8_t dispatch_code; +#endif #endif _PyEntryFrame entry; diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 05a2760671e847..c30638c221a03f 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -134,8 +134,8 @@ # define LABEL(name) name: #else # define TARGET(op) case op: TARGET_##op: -# define DISPATCH_GOTO() goto dispatch_opcode -# define DISPATCH_GOTO_NON_TRACING() goto dispatch_opcode +# define DISPATCH_GOTO() dispatch_code = opcode | tracing_mode ; goto dispatch_opcode +# define DISPATCH_GOTO_NON_TRACING() dispatch_code = opcode; goto dispatch_opcode # define JUMP_TO_LABEL(name) goto name; # define JUMP_TO_PREDICTED(name) goto PREDICTED_##name; # define LABEL(name) name: @@ -148,9 +148,9 @@ # define LEAVE_TRACING() \ DISPATCH_TABLE_VAR = DISPATCH_TABLE; #else -# define IS_JIT_TRACING() (0) -# define ENTER_TRACING() -# define LEAVE_TRACING() +# define IS_JIT_TRACING() (tracing_mode != 0) +# define ENTER_TRACING() tracing_mode = 255 +# define LEAVE_TRACING() tracing_mode = 0 #endif /* PRE_DISPATCH_GOTO() does lltrace if enabled. Normally a no-op */ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 6796abf84ac5f4..e1edd20b778d27 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7579,5 +7579,7 @@ break; } + /* _TRACE_RECORD is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ + #undef TIER_TWO diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 0d4678df68ce2d..b83b7c528e9150 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -11,7 +11,7 @@ #if !_Py_TAIL_CALL_INTERP #if !USE_COMPUTED_GOTOS dispatch_opcode: - switch (opcode) + switch (dispatch_code) #endif { #endif /* _Py_TAIL_CALL_INTERP */ @@ -11683,6 +11683,68 @@ DISPATCH(); } + TARGET(TRACE_RECORD) { + #if _Py_TAIL_CALL_INTERP + int opcode = TRACE_RECORD; + (void)(opcode); + #endif + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(TRACE_RECORD); + opcode = TRACE_RECORD; + #if _Py_TIER2 + assert(IS_JIT_TRACING()); + next_instr = this_instr; + frame->instr_ptr = prev_instr; + opcode = next_instr->op.code; + bool stop_tracing = (opcode == WITH_EXCEPT_START || + opcode == RERAISE || opcode == CLEANUP_THROW || + opcode == PUSH_EXC_INFO || opcode == INTERPRETER_EXIT); + _PyFrame_SetStackPointer(frame, stack_pointer); + int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing ? _DEOPT : 0); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (full) { + LEAVE_TRACING(); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = stop_tracing_and_jit(tstate, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + DISPATCH_GOTO_NON_TRACING(); + } + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + if ((_tstate->jit_tracer_state.prev_state.instr->op.code == CALL_LIST_APPEND && + opcode == POP_TOP) || + (_tstate->jit_tracer_state.prev_state.instr->op.code == BINARY_OP_INPLACE_ADD_UNICODE && + opcode == STORE_FAST)) { + _tstate->jit_tracer_state.prev_state.instr_is_super = true; + } + else { + _tstate->jit_tracer_state.prev_state.instr = next_instr; + } + PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable); + if (_tstate->jit_tracer_state.prev_state.instr_code != (PyCodeObject *)prev_code) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_SETREF(_tstate->jit_tracer_state.prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code))); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + _tstate->jit_tracer_state.prev_state.instr_frame = frame; + _tstate->jit_tracer_state.prev_state.instr_oparg = oparg; + _tstate->jit_tracer_state.prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + (&next_instr[1])->counter = trigger_backoff_counter(); + } + DISPATCH_GOTO_NON_TRACING(); + #else + (void)prev_instr; + Py_FatalError("JIT instruction executed in non-jit build."); + #endif + } + TARGET(UNARY_INVERT) { #if _Py_TAIL_CALL_INTERP int opcode = UNARY_INVERT; @@ -12254,55 +12316,6 @@ JUMP_TO_LABEL(error); DISPATCH(); } - LABEL(record_previous_inst) - { - #if _Py_TIER2 - assert(IS_JIT_TRACING()); - int opcode = next_instr->op.code; - bool stop_tracing = (opcode == WITH_EXCEPT_START || - opcode == RERAISE || opcode == CLEANUP_THROW || - opcode == PUSH_EXC_INFO || opcode == INTERPRETER_EXIT); - _PyFrame_SetStackPointer(frame, stack_pointer); - int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing ? _DEOPT : 0); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (full) { - LEAVE_TRACING(); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = stop_tracing_and_jit(tstate, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - JUMP_TO_LABEL(error); - } - DISPATCH_GOTO_NON_TRACING(); - } - _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; - if ((_tstate->jit_tracer_state.prev_state.instr->op.code == CALL_LIST_APPEND && - opcode == POP_TOP) || - (_tstate->jit_tracer_state.prev_state.instr->op.code == BINARY_OP_INPLACE_ADD_UNICODE && - opcode == STORE_FAST)) { - _tstate->jit_tracer_state.prev_state.instr_is_super = true; - } - else { - _tstate->jit_tracer_state.prev_state.instr = next_instr; - } - PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable); - if (_tstate->jit_tracer_state.prev_state.instr_code != (PyCodeObject *)prev_code) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_SETREF(_tstate->jit_tracer_state.prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code))); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - _tstate->jit_tracer_state.prev_state.instr_frame = frame; - _tstate->jit_tracer_state.prev_state.instr_oparg = oparg; - _tstate->jit_tracer_state.prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); - if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { - (&next_instr[1])->counter = trigger_backoff_counter(); - } - DISPATCH_GOTO_NON_TRACING(); - #else - Py_FatalError("JIT label executed in non-jit build."); - #endif - } - LABEL(stop_tracing) { #if _Py_TIER2 diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 81e46a331e0b9e..72b7433022fdea 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -191,7 +191,7 @@ is_instrumented(int opcode) { assert(opcode != 0); assert(opcode != RESERVED); - return opcode != ENTER_EXECUTOR && opcode >= MIN_INSTRUMENTED_OPCODE; + return opcode < ENTER_EXECUTOR && opcode >= MIN_INSTRUMENTED_OPCODE; } #ifndef NDEBUG @@ -526,7 +526,7 @@ valid_opcode(int opcode) if (IS_VALID_OPCODE(opcode) && opcode != CACHE && opcode != RESERVED && - opcode < 255) + opcode < 254) { return true; } diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 1b9196503b570b..b2fa7d01e8f6c2 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -233,7 +233,6 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_INSTRUMENTED_END_FOR, &&TARGET_INSTRUMENTED_POP_ITER, &&TARGET_INSTRUMENTED_END_SEND, @@ -256,220 +255,220 @@ static void *opcode_targets_table[256] = { &&TARGET_INSTRUMENTED_JUMP_BACKWARD, &&TARGET_INSTRUMENTED_LINE, &&TARGET_ENTER_EXECUTOR, + &&TARGET_TRACE_RECORD, }; #if _Py_TIER2 static void *opcode_tracing_targets_table[256] = { - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&_unknown_opcode, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, &&_unknown_opcode, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -493,28 +492,29 @@ static void *opcode_tracing_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, - &&record_previous_inst, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, }; #endif #else /* _Py_TAIL_CALL_INTERP */ @@ -528,7 +528,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_record_previous_inst(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_stop_tracing(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); @@ -746,6 +745,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACE_RECORD(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS); @@ -983,6 +983,7 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [TO_BOOL_LIST] = _TAIL_CALL_TO_BOOL_LIST, [TO_BOOL_NONE] = _TAIL_CALL_TO_BOOL_NONE, [TO_BOOL_STR] = _TAIL_CALL_TO_BOOL_STR, + [TRACE_RECORD] = _TAIL_CALL_TRACE_RECORD, [UNARY_INVERT] = _TAIL_CALL_UNARY_INVERT, [UNARY_NEGATIVE] = _TAIL_CALL_UNARY_NEGATIVE, [UNARY_NOT] = _TAIL_CALL_UNARY_NOT, @@ -1023,234 +1024,234 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [230] = _TAIL_CALL_UNKNOWN_OPCODE, [231] = _TAIL_CALL_UNKNOWN_OPCODE, [232] = _TAIL_CALL_UNKNOWN_OPCODE, - [233] = _TAIL_CALL_UNKNOWN_OPCODE, }; static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { - [BINARY_OP] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_ADD_INT] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_ADD_UNICODE] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_EXTEND] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_INPLACE_ADD_UNICODE] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_MULTIPLY_FLOAT] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_MULTIPLY_INT] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_SUBSCR_DICT] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_SUBSCR_GETITEM] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_SUBSCR_LIST_INT] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_SUBSCR_LIST_SLICE] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_SUBSCR_STR_INT] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_SUBSCR_TUPLE_INT] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_SUBTRACT_FLOAT] = _TAIL_CALL_record_previous_inst, - [BINARY_OP_SUBTRACT_INT] = _TAIL_CALL_record_previous_inst, - [BINARY_SLICE] = _TAIL_CALL_record_previous_inst, - [BUILD_INTERPOLATION] = _TAIL_CALL_record_previous_inst, - [BUILD_LIST] = _TAIL_CALL_record_previous_inst, - [BUILD_MAP] = _TAIL_CALL_record_previous_inst, - [BUILD_SET] = _TAIL_CALL_record_previous_inst, - [BUILD_SLICE] = _TAIL_CALL_record_previous_inst, - [BUILD_STRING] = _TAIL_CALL_record_previous_inst, - [BUILD_TEMPLATE] = _TAIL_CALL_record_previous_inst, - [BUILD_TUPLE] = _TAIL_CALL_record_previous_inst, - [CACHE] = _TAIL_CALL_record_previous_inst, - [CALL] = _TAIL_CALL_record_previous_inst, - [CALL_ALLOC_AND_ENTER_INIT] = _TAIL_CALL_record_previous_inst, - [CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_record_previous_inst, - [CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_record_previous_inst, - [CALL_BUILTIN_CLASS] = _TAIL_CALL_record_previous_inst, - [CALL_BUILTIN_FAST] = _TAIL_CALL_record_previous_inst, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = _TAIL_CALL_record_previous_inst, - [CALL_BUILTIN_O] = _TAIL_CALL_record_previous_inst, - [CALL_FUNCTION_EX] = _TAIL_CALL_record_previous_inst, - [CALL_INTRINSIC_1] = _TAIL_CALL_record_previous_inst, - [CALL_INTRINSIC_2] = _TAIL_CALL_record_previous_inst, - [CALL_ISINSTANCE] = _TAIL_CALL_record_previous_inst, - [CALL_KW] = _TAIL_CALL_record_previous_inst, - [CALL_KW_BOUND_METHOD] = _TAIL_CALL_record_previous_inst, - [CALL_KW_NON_PY] = _TAIL_CALL_record_previous_inst, - [CALL_KW_PY] = _TAIL_CALL_record_previous_inst, - [CALL_LEN] = _TAIL_CALL_record_previous_inst, - [CALL_LIST_APPEND] = _TAIL_CALL_record_previous_inst, - [CALL_METHOD_DESCRIPTOR_FAST] = _TAIL_CALL_record_previous_inst, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = _TAIL_CALL_record_previous_inst, - [CALL_METHOD_DESCRIPTOR_NOARGS] = _TAIL_CALL_record_previous_inst, - [CALL_METHOD_DESCRIPTOR_O] = _TAIL_CALL_record_previous_inst, - [CALL_NON_PY_GENERAL] = _TAIL_CALL_record_previous_inst, - [CALL_PY_EXACT_ARGS] = _TAIL_CALL_record_previous_inst, - [CALL_PY_GENERAL] = _TAIL_CALL_record_previous_inst, - [CALL_STR_1] = _TAIL_CALL_record_previous_inst, - [CALL_TUPLE_1] = _TAIL_CALL_record_previous_inst, - [CALL_TYPE_1] = _TAIL_CALL_record_previous_inst, - [CHECK_EG_MATCH] = _TAIL_CALL_record_previous_inst, - [CHECK_EXC_MATCH] = _TAIL_CALL_record_previous_inst, - [CLEANUP_THROW] = _TAIL_CALL_record_previous_inst, - [COMPARE_OP] = _TAIL_CALL_record_previous_inst, - [COMPARE_OP_FLOAT] = _TAIL_CALL_record_previous_inst, - [COMPARE_OP_INT] = _TAIL_CALL_record_previous_inst, - [COMPARE_OP_STR] = _TAIL_CALL_record_previous_inst, - [CONTAINS_OP] = _TAIL_CALL_record_previous_inst, - [CONTAINS_OP_DICT] = _TAIL_CALL_record_previous_inst, - [CONTAINS_OP_SET] = _TAIL_CALL_record_previous_inst, - [CONVERT_VALUE] = _TAIL_CALL_record_previous_inst, - [COPY] = _TAIL_CALL_record_previous_inst, - [COPY_FREE_VARS] = _TAIL_CALL_record_previous_inst, - [DELETE_ATTR] = _TAIL_CALL_record_previous_inst, - [DELETE_DEREF] = _TAIL_CALL_record_previous_inst, - [DELETE_FAST] = _TAIL_CALL_record_previous_inst, - [DELETE_GLOBAL] = _TAIL_CALL_record_previous_inst, - [DELETE_NAME] = _TAIL_CALL_record_previous_inst, - [DELETE_SUBSCR] = _TAIL_CALL_record_previous_inst, - [DICT_MERGE] = _TAIL_CALL_record_previous_inst, - [DICT_UPDATE] = _TAIL_CALL_record_previous_inst, - [END_ASYNC_FOR] = _TAIL_CALL_record_previous_inst, - [END_FOR] = _TAIL_CALL_record_previous_inst, - [END_SEND] = _TAIL_CALL_record_previous_inst, - [ENTER_EXECUTOR] = _TAIL_CALL_record_previous_inst, - [EXIT_INIT_CHECK] = _TAIL_CALL_record_previous_inst, - [EXTENDED_ARG] = _TAIL_CALL_record_previous_inst, - [FORMAT_SIMPLE] = _TAIL_CALL_record_previous_inst, - [FORMAT_WITH_SPEC] = _TAIL_CALL_record_previous_inst, - [FOR_ITER] = _TAIL_CALL_record_previous_inst, - [FOR_ITER_GEN] = _TAIL_CALL_record_previous_inst, - [FOR_ITER_LIST] = _TAIL_CALL_record_previous_inst, - [FOR_ITER_RANGE] = _TAIL_CALL_record_previous_inst, - [FOR_ITER_TUPLE] = _TAIL_CALL_record_previous_inst, - [GET_AITER] = _TAIL_CALL_record_previous_inst, - [GET_ANEXT] = _TAIL_CALL_record_previous_inst, - [GET_AWAITABLE] = _TAIL_CALL_record_previous_inst, - [GET_ITER] = _TAIL_CALL_record_previous_inst, - [GET_LEN] = _TAIL_CALL_record_previous_inst, - [GET_YIELD_FROM_ITER] = _TAIL_CALL_record_previous_inst, - [IMPORT_FROM] = _TAIL_CALL_record_previous_inst, - [IMPORT_NAME] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_CALL] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_CALL_FUNCTION_EX] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_CALL_KW] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_END_ASYNC_FOR] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_END_FOR] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_END_SEND] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_FOR_ITER] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_INSTRUCTION] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_JUMP_BACKWARD] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_JUMP_FORWARD] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_LINE] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_LOAD_SUPER_ATTR] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_NOT_TAKEN] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_POP_ITER] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_POP_JUMP_IF_FALSE] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_POP_JUMP_IF_NONE] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_POP_JUMP_IF_TRUE] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_RESUME] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_RETURN_VALUE] = _TAIL_CALL_record_previous_inst, - [INSTRUMENTED_YIELD_VALUE] = _TAIL_CALL_record_previous_inst, - [INTERPRETER_EXIT] = _TAIL_CALL_record_previous_inst, - [IS_OP] = _TAIL_CALL_record_previous_inst, - [JUMP_BACKWARD] = _TAIL_CALL_record_previous_inst, - [JUMP_BACKWARD_JIT] = _TAIL_CALL_record_previous_inst, - [JUMP_BACKWARD_NO_INTERRUPT] = _TAIL_CALL_record_previous_inst, - [JUMP_BACKWARD_NO_JIT] = _TAIL_CALL_record_previous_inst, - [JUMP_FORWARD] = _TAIL_CALL_record_previous_inst, - [LIST_APPEND] = _TAIL_CALL_record_previous_inst, - [LIST_EXTEND] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_CLASS] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_INSTANCE_VALUE] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_METHOD_LAZY_DICT] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_METHOD_NO_DICT] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_METHOD_WITH_VALUES] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_MODULE] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_PROPERTY] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_SLOT] = _TAIL_CALL_record_previous_inst, - [LOAD_ATTR_WITH_HINT] = _TAIL_CALL_record_previous_inst, - [LOAD_BUILD_CLASS] = _TAIL_CALL_record_previous_inst, - [LOAD_COMMON_CONSTANT] = _TAIL_CALL_record_previous_inst, - [LOAD_CONST] = _TAIL_CALL_record_previous_inst, - [LOAD_DEREF] = _TAIL_CALL_record_previous_inst, - [LOAD_FAST] = _TAIL_CALL_record_previous_inst, - [LOAD_FAST_AND_CLEAR] = _TAIL_CALL_record_previous_inst, - [LOAD_FAST_BORROW] = _TAIL_CALL_record_previous_inst, - [LOAD_FAST_BORROW_LOAD_FAST_BORROW] = _TAIL_CALL_record_previous_inst, - [LOAD_FAST_CHECK] = _TAIL_CALL_record_previous_inst, - [LOAD_FAST_LOAD_FAST] = _TAIL_CALL_record_previous_inst, - [LOAD_FROM_DICT_OR_DEREF] = _TAIL_CALL_record_previous_inst, - [LOAD_FROM_DICT_OR_GLOBALS] = _TAIL_CALL_record_previous_inst, - [LOAD_GLOBAL] = _TAIL_CALL_record_previous_inst, - [LOAD_GLOBAL_BUILTIN] = _TAIL_CALL_record_previous_inst, - [LOAD_GLOBAL_MODULE] = _TAIL_CALL_record_previous_inst, - [LOAD_LOCALS] = _TAIL_CALL_record_previous_inst, - [LOAD_NAME] = _TAIL_CALL_record_previous_inst, - [LOAD_SMALL_INT] = _TAIL_CALL_record_previous_inst, - [LOAD_SPECIAL] = _TAIL_CALL_record_previous_inst, - [LOAD_SUPER_ATTR] = _TAIL_CALL_record_previous_inst, - [LOAD_SUPER_ATTR_ATTR] = _TAIL_CALL_record_previous_inst, - [LOAD_SUPER_ATTR_METHOD] = _TAIL_CALL_record_previous_inst, - [MAKE_CELL] = _TAIL_CALL_record_previous_inst, - [MAKE_FUNCTION] = _TAIL_CALL_record_previous_inst, - [MAP_ADD] = _TAIL_CALL_record_previous_inst, - [MATCH_CLASS] = _TAIL_CALL_record_previous_inst, - [MATCH_KEYS] = _TAIL_CALL_record_previous_inst, - [MATCH_MAPPING] = _TAIL_CALL_record_previous_inst, - [MATCH_SEQUENCE] = _TAIL_CALL_record_previous_inst, - [NOP] = _TAIL_CALL_record_previous_inst, - [NOT_TAKEN] = _TAIL_CALL_record_previous_inst, - [POP_EXCEPT] = _TAIL_CALL_record_previous_inst, - [POP_ITER] = _TAIL_CALL_record_previous_inst, - [POP_JUMP_IF_FALSE] = _TAIL_CALL_record_previous_inst, - [POP_JUMP_IF_NONE] = _TAIL_CALL_record_previous_inst, - [POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_record_previous_inst, - [POP_JUMP_IF_TRUE] = _TAIL_CALL_record_previous_inst, - [POP_TOP] = _TAIL_CALL_record_previous_inst, - [PUSH_EXC_INFO] = _TAIL_CALL_record_previous_inst, - [PUSH_NULL] = _TAIL_CALL_record_previous_inst, - [RAISE_VARARGS] = _TAIL_CALL_record_previous_inst, - [RERAISE] = _TAIL_CALL_record_previous_inst, - [RESERVED] = _TAIL_CALL_record_previous_inst, - [RESUME] = _TAIL_CALL_record_previous_inst, - [RESUME_CHECK] = _TAIL_CALL_record_previous_inst, - [RETURN_GENERATOR] = _TAIL_CALL_record_previous_inst, - [RETURN_VALUE] = _TAIL_CALL_record_previous_inst, - [SEND] = _TAIL_CALL_record_previous_inst, - [SEND_GEN] = _TAIL_CALL_record_previous_inst, - [SETUP_ANNOTATIONS] = _TAIL_CALL_record_previous_inst, - [SET_ADD] = _TAIL_CALL_record_previous_inst, - [SET_FUNCTION_ATTRIBUTE] = _TAIL_CALL_record_previous_inst, - [SET_UPDATE] = _TAIL_CALL_record_previous_inst, - [STORE_ATTR] = _TAIL_CALL_record_previous_inst, - [STORE_ATTR_INSTANCE_VALUE] = _TAIL_CALL_record_previous_inst, - [STORE_ATTR_SLOT] = _TAIL_CALL_record_previous_inst, - [STORE_ATTR_WITH_HINT] = _TAIL_CALL_record_previous_inst, - [STORE_DEREF] = _TAIL_CALL_record_previous_inst, - [STORE_FAST] = _TAIL_CALL_record_previous_inst, - [STORE_FAST_LOAD_FAST] = _TAIL_CALL_record_previous_inst, - [STORE_FAST_STORE_FAST] = _TAIL_CALL_record_previous_inst, - [STORE_GLOBAL] = _TAIL_CALL_record_previous_inst, - [STORE_NAME] = _TAIL_CALL_record_previous_inst, - [STORE_SLICE] = _TAIL_CALL_record_previous_inst, - [STORE_SUBSCR] = _TAIL_CALL_record_previous_inst, - [STORE_SUBSCR_DICT] = _TAIL_CALL_record_previous_inst, - [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_record_previous_inst, - [SWAP] = _TAIL_CALL_record_previous_inst, - [TO_BOOL] = _TAIL_CALL_record_previous_inst, - [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_record_previous_inst, - [TO_BOOL_BOOL] = _TAIL_CALL_record_previous_inst, - [TO_BOOL_INT] = _TAIL_CALL_record_previous_inst, - [TO_BOOL_LIST] = _TAIL_CALL_record_previous_inst, - [TO_BOOL_NONE] = _TAIL_CALL_record_previous_inst, - [TO_BOOL_STR] = _TAIL_CALL_record_previous_inst, - [UNARY_INVERT] = _TAIL_CALL_record_previous_inst, - [UNARY_NEGATIVE] = _TAIL_CALL_record_previous_inst, - [UNARY_NOT] = _TAIL_CALL_record_previous_inst, - [UNPACK_EX] = _TAIL_CALL_record_previous_inst, - [UNPACK_SEQUENCE] = _TAIL_CALL_record_previous_inst, - [UNPACK_SEQUENCE_LIST] = _TAIL_CALL_record_previous_inst, - [UNPACK_SEQUENCE_TUPLE] = _TAIL_CALL_record_previous_inst, - [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_record_previous_inst, - [WITH_EXCEPT_START] = _TAIL_CALL_record_previous_inst, - [YIELD_VALUE] = _TAIL_CALL_record_previous_inst, + [BINARY_OP] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_ADD_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_ADD_UNICODE] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_EXTEND] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_INPLACE_ADD_UNICODE] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_MULTIPLY_FLOAT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_MULTIPLY_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_DICT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_GETITEM] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_LIST_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_LIST_SLICE] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_STR_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBSCR_TUPLE_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBTRACT_FLOAT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_OP_SUBTRACT_INT] = _TAIL_CALL_TRACE_RECORD, + [BINARY_SLICE] = _TAIL_CALL_TRACE_RECORD, + [BUILD_INTERPOLATION] = _TAIL_CALL_TRACE_RECORD, + [BUILD_LIST] = _TAIL_CALL_TRACE_RECORD, + [BUILD_MAP] = _TAIL_CALL_TRACE_RECORD, + [BUILD_SET] = _TAIL_CALL_TRACE_RECORD, + [BUILD_SLICE] = _TAIL_CALL_TRACE_RECORD, + [BUILD_STRING] = _TAIL_CALL_TRACE_RECORD, + [BUILD_TEMPLATE] = _TAIL_CALL_TRACE_RECORD, + [BUILD_TUPLE] = _TAIL_CALL_TRACE_RECORD, + [CACHE] = _TAIL_CALL_TRACE_RECORD, + [CALL] = _TAIL_CALL_TRACE_RECORD, + [CALL_ALLOC_AND_ENTER_INIT] = _TAIL_CALL_TRACE_RECORD, + [CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_TRACE_RECORD, + [CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_TRACE_RECORD, + [CALL_BUILTIN_CLASS] = _TAIL_CALL_TRACE_RECORD, + [CALL_BUILTIN_FAST] = _TAIL_CALL_TRACE_RECORD, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = _TAIL_CALL_TRACE_RECORD, + [CALL_BUILTIN_O] = _TAIL_CALL_TRACE_RECORD, + [CALL_FUNCTION_EX] = _TAIL_CALL_TRACE_RECORD, + [CALL_INTRINSIC_1] = _TAIL_CALL_TRACE_RECORD, + [CALL_INTRINSIC_2] = _TAIL_CALL_TRACE_RECORD, + [CALL_ISINSTANCE] = _TAIL_CALL_TRACE_RECORD, + [CALL_KW] = _TAIL_CALL_TRACE_RECORD, + [CALL_KW_BOUND_METHOD] = _TAIL_CALL_TRACE_RECORD, + [CALL_KW_NON_PY] = _TAIL_CALL_TRACE_RECORD, + [CALL_KW_PY] = _TAIL_CALL_TRACE_RECORD, + [CALL_LEN] = _TAIL_CALL_TRACE_RECORD, + [CALL_LIST_APPEND] = _TAIL_CALL_TRACE_RECORD, + [CALL_METHOD_DESCRIPTOR_FAST] = _TAIL_CALL_TRACE_RECORD, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = _TAIL_CALL_TRACE_RECORD, + [CALL_METHOD_DESCRIPTOR_NOARGS] = _TAIL_CALL_TRACE_RECORD, + [CALL_METHOD_DESCRIPTOR_O] = _TAIL_CALL_TRACE_RECORD, + [CALL_NON_PY_GENERAL] = _TAIL_CALL_TRACE_RECORD, + [CALL_PY_EXACT_ARGS] = _TAIL_CALL_TRACE_RECORD, + [CALL_PY_GENERAL] = _TAIL_CALL_TRACE_RECORD, + [CALL_STR_1] = _TAIL_CALL_TRACE_RECORD, + [CALL_TUPLE_1] = _TAIL_CALL_TRACE_RECORD, + [CALL_TYPE_1] = _TAIL_CALL_TRACE_RECORD, + [CHECK_EG_MATCH] = _TAIL_CALL_TRACE_RECORD, + [CHECK_EXC_MATCH] = _TAIL_CALL_TRACE_RECORD, + [CLEANUP_THROW] = _TAIL_CALL_TRACE_RECORD, + [COMPARE_OP] = _TAIL_CALL_TRACE_RECORD, + [COMPARE_OP_FLOAT] = _TAIL_CALL_TRACE_RECORD, + [COMPARE_OP_INT] = _TAIL_CALL_TRACE_RECORD, + [COMPARE_OP_STR] = _TAIL_CALL_TRACE_RECORD, + [CONTAINS_OP] = _TAIL_CALL_TRACE_RECORD, + [CONTAINS_OP_DICT] = _TAIL_CALL_TRACE_RECORD, + [CONTAINS_OP_SET] = _TAIL_CALL_TRACE_RECORD, + [CONVERT_VALUE] = _TAIL_CALL_TRACE_RECORD, + [COPY] = _TAIL_CALL_TRACE_RECORD, + [COPY_FREE_VARS] = _TAIL_CALL_TRACE_RECORD, + [DELETE_ATTR] = _TAIL_CALL_TRACE_RECORD, + [DELETE_DEREF] = _TAIL_CALL_TRACE_RECORD, + [DELETE_FAST] = _TAIL_CALL_TRACE_RECORD, + [DELETE_GLOBAL] = _TAIL_CALL_TRACE_RECORD, + [DELETE_NAME] = _TAIL_CALL_TRACE_RECORD, + [DELETE_SUBSCR] = _TAIL_CALL_TRACE_RECORD, + [DICT_MERGE] = _TAIL_CALL_TRACE_RECORD, + [DICT_UPDATE] = _TAIL_CALL_TRACE_RECORD, + [END_ASYNC_FOR] = _TAIL_CALL_TRACE_RECORD, + [END_FOR] = _TAIL_CALL_TRACE_RECORD, + [END_SEND] = _TAIL_CALL_TRACE_RECORD, + [ENTER_EXECUTOR] = _TAIL_CALL_TRACE_RECORD, + [EXIT_INIT_CHECK] = _TAIL_CALL_TRACE_RECORD, + [EXTENDED_ARG] = _TAIL_CALL_TRACE_RECORD, + [FORMAT_SIMPLE] = _TAIL_CALL_TRACE_RECORD, + [FORMAT_WITH_SPEC] = _TAIL_CALL_TRACE_RECORD, + [FOR_ITER] = _TAIL_CALL_TRACE_RECORD, + [FOR_ITER_GEN] = _TAIL_CALL_TRACE_RECORD, + [FOR_ITER_LIST] = _TAIL_CALL_TRACE_RECORD, + [FOR_ITER_RANGE] = _TAIL_CALL_TRACE_RECORD, + [FOR_ITER_TUPLE] = _TAIL_CALL_TRACE_RECORD, + [GET_AITER] = _TAIL_CALL_TRACE_RECORD, + [GET_ANEXT] = _TAIL_CALL_TRACE_RECORD, + [GET_AWAITABLE] = _TAIL_CALL_TRACE_RECORD, + [GET_ITER] = _TAIL_CALL_TRACE_RECORD, + [GET_LEN] = _TAIL_CALL_TRACE_RECORD, + [GET_YIELD_FROM_ITER] = _TAIL_CALL_TRACE_RECORD, + [IMPORT_FROM] = _TAIL_CALL_TRACE_RECORD, + [IMPORT_NAME] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_CALL] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_CALL_FUNCTION_EX] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_CALL_KW] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_END_ASYNC_FOR] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_END_FOR] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_END_SEND] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_FOR_ITER] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_INSTRUCTION] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_JUMP_BACKWARD] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_JUMP_FORWARD] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_LINE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_LOAD_SUPER_ATTR] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_NOT_TAKEN] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_POP_ITER] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_POP_JUMP_IF_NONE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_RESUME] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_RETURN_VALUE] = _TAIL_CALL_TRACE_RECORD, + [INSTRUMENTED_YIELD_VALUE] = _TAIL_CALL_TRACE_RECORD, + [INTERPRETER_EXIT] = _TAIL_CALL_TRACE_RECORD, + [IS_OP] = _TAIL_CALL_TRACE_RECORD, + [JUMP_BACKWARD] = _TAIL_CALL_TRACE_RECORD, + [JUMP_BACKWARD_JIT] = _TAIL_CALL_TRACE_RECORD, + [JUMP_BACKWARD_NO_INTERRUPT] = _TAIL_CALL_TRACE_RECORD, + [JUMP_BACKWARD_NO_JIT] = _TAIL_CALL_TRACE_RECORD, + [JUMP_FORWARD] = _TAIL_CALL_TRACE_RECORD, + [LIST_APPEND] = _TAIL_CALL_TRACE_RECORD, + [LIST_EXTEND] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_CLASS] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_INSTANCE_VALUE] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_METHOD_LAZY_DICT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_METHOD_NO_DICT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_METHOD_WITH_VALUES] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_MODULE] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_PROPERTY] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_SLOT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_ATTR_WITH_HINT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_BUILD_CLASS] = _TAIL_CALL_TRACE_RECORD, + [LOAD_COMMON_CONSTANT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_CONST] = _TAIL_CALL_TRACE_RECORD, + [LOAD_DEREF] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST_AND_CLEAR] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST_BORROW] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST_BORROW_LOAD_FAST_BORROW] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST_CHECK] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FAST_LOAD_FAST] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FROM_DICT_OR_DEREF] = _TAIL_CALL_TRACE_RECORD, + [LOAD_FROM_DICT_OR_GLOBALS] = _TAIL_CALL_TRACE_RECORD, + [LOAD_GLOBAL] = _TAIL_CALL_TRACE_RECORD, + [LOAD_GLOBAL_BUILTIN] = _TAIL_CALL_TRACE_RECORD, + [LOAD_GLOBAL_MODULE] = _TAIL_CALL_TRACE_RECORD, + [LOAD_LOCALS] = _TAIL_CALL_TRACE_RECORD, + [LOAD_NAME] = _TAIL_CALL_TRACE_RECORD, + [LOAD_SMALL_INT] = _TAIL_CALL_TRACE_RECORD, + [LOAD_SPECIAL] = _TAIL_CALL_TRACE_RECORD, + [LOAD_SUPER_ATTR] = _TAIL_CALL_TRACE_RECORD, + [LOAD_SUPER_ATTR_ATTR] = _TAIL_CALL_TRACE_RECORD, + [LOAD_SUPER_ATTR_METHOD] = _TAIL_CALL_TRACE_RECORD, + [MAKE_CELL] = _TAIL_CALL_TRACE_RECORD, + [MAKE_FUNCTION] = _TAIL_CALL_TRACE_RECORD, + [MAP_ADD] = _TAIL_CALL_TRACE_RECORD, + [MATCH_CLASS] = _TAIL_CALL_TRACE_RECORD, + [MATCH_KEYS] = _TAIL_CALL_TRACE_RECORD, + [MATCH_MAPPING] = _TAIL_CALL_TRACE_RECORD, + [MATCH_SEQUENCE] = _TAIL_CALL_TRACE_RECORD, + [NOP] = _TAIL_CALL_TRACE_RECORD, + [NOT_TAKEN] = _TAIL_CALL_TRACE_RECORD, + [POP_EXCEPT] = _TAIL_CALL_TRACE_RECORD, + [POP_ITER] = _TAIL_CALL_TRACE_RECORD, + [POP_JUMP_IF_FALSE] = _TAIL_CALL_TRACE_RECORD, + [POP_JUMP_IF_NONE] = _TAIL_CALL_TRACE_RECORD, + [POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_TRACE_RECORD, + [POP_JUMP_IF_TRUE] = _TAIL_CALL_TRACE_RECORD, + [POP_TOP] = _TAIL_CALL_TRACE_RECORD, + [PUSH_EXC_INFO] = _TAIL_CALL_TRACE_RECORD, + [PUSH_NULL] = _TAIL_CALL_TRACE_RECORD, + [RAISE_VARARGS] = _TAIL_CALL_TRACE_RECORD, + [RERAISE] = _TAIL_CALL_TRACE_RECORD, + [RESERVED] = _TAIL_CALL_TRACE_RECORD, + [RESUME] = _TAIL_CALL_TRACE_RECORD, + [RESUME_CHECK] = _TAIL_CALL_TRACE_RECORD, + [RETURN_GENERATOR] = _TAIL_CALL_TRACE_RECORD, + [RETURN_VALUE] = _TAIL_CALL_TRACE_RECORD, + [SEND] = _TAIL_CALL_TRACE_RECORD, + [SEND_GEN] = _TAIL_CALL_TRACE_RECORD, + [SETUP_ANNOTATIONS] = _TAIL_CALL_TRACE_RECORD, + [SET_ADD] = _TAIL_CALL_TRACE_RECORD, + [SET_FUNCTION_ATTRIBUTE] = _TAIL_CALL_TRACE_RECORD, + [SET_UPDATE] = _TAIL_CALL_TRACE_RECORD, + [STORE_ATTR] = _TAIL_CALL_TRACE_RECORD, + [STORE_ATTR_INSTANCE_VALUE] = _TAIL_CALL_TRACE_RECORD, + [STORE_ATTR_SLOT] = _TAIL_CALL_TRACE_RECORD, + [STORE_ATTR_WITH_HINT] = _TAIL_CALL_TRACE_RECORD, + [STORE_DEREF] = _TAIL_CALL_TRACE_RECORD, + [STORE_FAST] = _TAIL_CALL_TRACE_RECORD, + [STORE_FAST_LOAD_FAST] = _TAIL_CALL_TRACE_RECORD, + [STORE_FAST_STORE_FAST] = _TAIL_CALL_TRACE_RECORD, + [STORE_GLOBAL] = _TAIL_CALL_TRACE_RECORD, + [STORE_NAME] = _TAIL_CALL_TRACE_RECORD, + [STORE_SLICE] = _TAIL_CALL_TRACE_RECORD, + [STORE_SUBSCR] = _TAIL_CALL_TRACE_RECORD, + [STORE_SUBSCR_DICT] = _TAIL_CALL_TRACE_RECORD, + [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_TRACE_RECORD, + [SWAP] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_BOOL] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_INT] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_LIST] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_NONE] = _TAIL_CALL_TRACE_RECORD, + [TO_BOOL_STR] = _TAIL_CALL_TRACE_RECORD, + [TRACE_RECORD] = _TAIL_CALL_TRACE_RECORD, + [UNARY_INVERT] = _TAIL_CALL_TRACE_RECORD, + [UNARY_NEGATIVE] = _TAIL_CALL_TRACE_RECORD, + [UNARY_NOT] = _TAIL_CALL_TRACE_RECORD, + [UNPACK_EX] = _TAIL_CALL_TRACE_RECORD, + [UNPACK_SEQUENCE] = _TAIL_CALL_TRACE_RECORD, + [UNPACK_SEQUENCE_LIST] = _TAIL_CALL_TRACE_RECORD, + [UNPACK_SEQUENCE_TUPLE] = _TAIL_CALL_TRACE_RECORD, + [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_TRACE_RECORD, + [WITH_EXCEPT_START] = _TAIL_CALL_TRACE_RECORD, + [YIELD_VALUE] = _TAIL_CALL_TRACE_RECORD, [121] = _TAIL_CALL_UNKNOWN_OPCODE, [122] = _TAIL_CALL_UNKNOWN_OPCODE, [123] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1281,6 +1282,5 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [230] = _TAIL_CALL_UNKNOWN_OPCODE, [231] = _TAIL_CALL_UNKNOWN_OPCODE, [232] = _TAIL_CALL_UNKNOWN_OPCODE, - [233] = _TAIL_CALL_UNKNOWN_OPCODE, }; #endif /* _Py_TAIL_CALL_INTERP */ diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 01263fe8c7a78f..9ebd113df2dabf 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -3483,3 +3483,5 @@ break; } + /* _TRACE_RECORD is not a viable micro-op for tier 2 */ + diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index d39013db4f7fd6..93aa4899fe6ec8 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -1195,8 +1195,9 @@ def assign_opcodes( # This is an historical oddity. instmap["BINARY_OP_INPLACE_ADD_UNICODE"] = 3 - instmap["INSTRUMENTED_LINE"] = 254 - instmap["ENTER_EXECUTOR"] = 255 + instmap["INSTRUMENTED_LINE"] = 253 + instmap["ENTER_EXECUTOR"] = 254 + instmap["TRACE_RECORD"] = 255 instrumented = [name for name in instructions if name.startswith("INSTRUMENTED")] @@ -1221,7 +1222,7 @@ def assign_opcodes( # Specialized ops appear in their own section # Instrumented opcodes are at the end of the valid range min_internal = instmap["RESUME"] + 1 - min_instrumented = 254 - (len(instrumented) - 1) + min_instrumented = 254 - len(instrumented) assert min_internal + len(specialized) < min_instrumented next_opcode = 1 diff --git a/Tools/cases_generator/target_generator.py b/Tools/cases_generator/target_generator.py index 36fa1d7fa4908b..f633f704485819 100644 --- a/Tools/cases_generator/target_generator.py +++ b/Tools/cases_generator/target_generator.py @@ -34,7 +34,7 @@ def write_opcode_targets(analysis: Analysis, out: CWriter) -> None: targets = ["&&_unknown_opcode,\n"] * 256 for name, op in analysis.opmap.items(): if op < 256: - targets[op] = f"&&record_previous_inst,\n" + targets[op] = f"&&TARGET_TRACE_RECORD,\n" out.emit("#if _Py_TIER2\n") out.emit("static void *opcode_tracing_targets_table[256] = {\n") for target in targets: @@ -84,7 +84,7 @@ def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: # Emit the tracing dispatch table. out.emit("static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = {\n") for name in sorted(analysis.instructions.keys()): - out.emit(f"[{name}] = _TAIL_CALL_record_previous_inst,\n") + out.emit(f"[{name}] = _TAIL_CALL_TRACE_RECORD,\n") named_values = analysis.opmap.values() for rest in range(256): if rest not in named_values: diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 94ffb0118f0786..c7ff5de681e6fa 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -160,7 +160,7 @@ def generate_tier1( #if !_Py_TAIL_CALL_INTERP #if !USE_COMPUTED_GOTOS dispatch_opcode: - switch (opcode) + switch (dispatch_code) #endif {{ #endif /* _Py_TAIL_CALL_INTERP */ From f46785f8bc118e0efb840af1e520777b1baa03d9 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 18 Nov 2025 16:42:13 +0200 Subject: [PATCH 089/638] gh-133879: Copyedit "What's new in Python 3.15" (#141717) --- .../pending-removal-in-future.rst | 2 +- Doc/whatsnew/3.15.rst | 36 +++++++++---------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Doc/deprecations/pending-removal-in-future.rst b/Doc/deprecations/pending-removal-in-future.rst index 7ed430625f305e..301867416701ea 100644 --- a/Doc/deprecations/pending-removal-in-future.rst +++ b/Doc/deprecations/pending-removal-in-future.rst @@ -76,7 +76,7 @@ although there is currently no date scheduled for their removal. * :mod:`mailbox`: Use of StringIO input and text mode is deprecated, use BytesIO and binary mode instead. -* :mod:`os`: Calling :func:`os.register_at_fork` in multi-threaded process. +* :mod:`os`: Calling :func:`os.register_at_fork` in a multi-threaded process. * :class:`!pydoc.ErrorDuringImport`: A tuple value for *exc_info* parameter is deprecated, use an exception instance. diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index cf5bef15203b23..24cc7e2d7eb911 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -316,9 +316,7 @@ Other language changes and compression. Common code patterns which can be optimized with :func:`~bytearray.take_bytes` are listed below. - (Contributed by Cody Maloney in :gh:`139871`.) - - .. list-table:: Suggested Optimizing Refactors + .. list-table:: Suggested optimizing refactors :header-rows: 1 * - Description @@ -387,10 +385,12 @@ Other language changes buffer.resize(n) data = buffer.take_bytes() + (Contributed by Cody Maloney in :gh:`139871`.) + * Many functions related to compiling or parsing Python code, such as :func:`compile`, :func:`ast.parse`, :func:`symtable.symtable`, - and :func:`importlib.abc.InspectLoader.source_to_code`, now allow to pass - the module name. It is needed to unambiguous :ref:`filter ` + and :func:`importlib.abc.InspectLoader.source_to_code`, now allow the module + name to be passed. It is needed to unambiguously :ref:`filter ` syntax warnings by module name. (Contributed by Serhiy Storchaka in :gh:`135801`.) @@ -776,6 +776,17 @@ unittest (Contributed by Garry Cairns in :gh:`134567`.) +venv +---- + +* On POSIX platforms, platlib directories will be created if needed when + creating virtual environments, instead of using ``lib64 -> lib`` symlink. + This means purelib and platlib of virtual environments no longer share the + same ``lib`` directory on platforms where :data:`sys.platlibdir` is not + equal to ``lib``. + (Contributed by Rui Xi in :gh:`133951`.) + + warnings -------- @@ -788,17 +799,6 @@ warnings (Contributed by Serhiy Storchaka in :gh:`135801`.) -venv ----- - -* On POSIX platforms, platlib directories will be created if needed when - creating virtual environments, instead of using ``lib64 -> lib`` symlink. - This means purelib and platlib of virtual environments no longer share the - same ``lib`` directory on platforms where :data:`sys.platlibdir` is not - equal to ``lib``. - (Contributed by Rui Xi in :gh:`133951`.) - - xml.parsers.expat ----------------- @@ -1242,7 +1242,7 @@ Porting to Python 3.15 This section lists previously described changes and other bugfixes that may require changes to your code. -* :class:`sqlite3.Connection` APIs has been cleaned up. +* :class:`sqlite3.Connection` APIs have been cleaned up. * All parameters of :func:`sqlite3.connect` except *database* are now keyword-only. * The first three parameters of methods :meth:`~sqlite3.Connection.create_function` @@ -1262,7 +1262,7 @@ that may require changes to your code. * :meth:`~mmap.mmap.resize` has been removed on platforms that don't support the underlying syscall, instead of raising a :exc:`SystemError`. -* Resource warning is now emitted for unclosed +* A resource warning is now emitted for an unclosed :func:`xml.etree.ElementTree.iterparse` iterator if it opened a file. Use its :meth:`!close` method or the :func:`contextlib.closing` context manager to close it. From a62562859deea162a36dd5c99f0b87fe09af0292 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 18 Nov 2025 16:50:49 +0200 Subject: [PATCH 090/638] Python 3.15.0a2 --- Doc/c-api/import.rst | 2 +- Doc/c-api/init.rst | 4 +- Doc/library/ast.rst | 2 +- Doc/library/decimal.rst | 2 +- Doc/library/functions.rst | 2 +- Doc/library/functools.rst | 2 +- Doc/library/importlib.rst | 4 +- Doc/library/inspect.rst | 2 +- Doc/library/math.integer.rst | 2 +- Doc/library/math.rst | 2 +- Doc/library/os.rst | 14 +- Doc/library/stat.rst | 2 +- Doc/library/stdtypes.rst | 2 +- Doc/library/symtable.rst | 2 +- Doc/library/unicodedata.rst | 4 +- Doc/library/warnings.rst | 2 +- Doc/library/winreg.rst | 2 +- Doc/library/xml.etree.elementtree.rst | 2 +- Include/patchlevel.h | 4 +- Lib/pydoc_data/topics.py | 51 +- Misc/NEWS.d/3.15.0a2.rst | 1746 +++++++++++++++++ ...-08-10-22-28-06.gh-issue-137618.FdNvIE.rst | 2 - ...-10-16-11-30-53.gh-issue-140189.YCrUyt.rst | 1 - ...-10-17-11-33-45.gh-issue-140239._k-GgW.rst | 1 - ...-10-22-12-44-07.gh-issue-140475.OhzQbR.rst | 1 - ...-10-25-08-07-06.gh-issue-140513.6OhLTs.rst | 2 - ...-10-29-12-30-38.gh-issue-140768.ITYrzw.rst | 1 - ...-10-31-13-20-16.gh-issue-140454.gF6dCe.rst | 3 - ...-10-06-22-17-47.gh-issue-139653.6-1MOd.rst | 4 - ...-10-15-15-59-59.gh-issue-140153.BO7sH4.rst | 2 - ...-10-26-16-45-06.gh-issue-140487.fGOqss.rst | 2 - ...-10-26-16-45-28.gh-issue-140556.s__Dae.rst | 2 - ...-11-05-04-38-16.gh-issue-141004.rJL43P.rst | 1 - ...-11-05-05-45-49.gh-issue-141004.N9Ooh9.rst | 1 - ...-11-06-06-28-14.gh-issue-141042.brOioJ.rst | 3 - ...-11-08-10-51-50.gh-issue-116146.pCmx6L.rst | 2 - ...-11-10-11-26-26.gh-issue-141341.OsO6-y.rst | 2 - ...-06-24-13-12-58.gh-issue-134786.MF0VVk.rst | 2 - ...-07-08-00-41-46.gh-issue-136327.7AiTb_.rst | 2 - ...-07-29-17-51-14.gh-issue-131253.GpRjWy.rst | 1 - ...-09-13-01-23-25.gh-issue-138857.YQ5gdc.rst | 2 - ...-09-15-13-06-11.gh-issue-138944.PeCgLb.rst | 3 - ...-09-23-21-01-12.gh-issue-139269.1rIaxy.rst | 1 - ...-10-03-17-51-43.gh-issue-139475._684ED.rst | 2 - ...-10-06-10-03-37.gh-issue-139640.gY5oTb.rst | 3 - ...10-06-10-03-37.gh-issue-139640.gY5oTb2.rst | 3 - ...-10-06-14-19-47.gh-issue-135801.OhxEZS.rst | 6 - ...-10-12-01-12-12.gh-issue-139817.PAn-8Z.rst | 2 - ...-10-13-13-54-19.gh-issue-139914.M-y_3E.rst | 1 - ...-10-14-17-07-37.gh-issue-140067.ID2gOm.rst | 1 - ...-10-14-18-24-16.gh-issue-139871.SWtuUz.rst | 2 - ...-10-14-20-18-31.gh-issue-140080.8ROjxW.rst | 1 - ...-10-15-00-21-40.gh-issue-140061.J0XeDV.rst | 2 - ...-10-15-17-12-32.gh-issue-140149.cy1m3d.rst | 2 - ...-10-16-21-47-00.gh-issue-140104.A8SQIm.rst | 2 - ...-10-17-14-38-10.gh-issue-140253.gCqFaL.rst | 2 - ...-10-17-18-03-12.gh-issue-139951.IdwM2O.rst | 7 - ...-10-17-20-23-19.gh-issue-140257.8Txmem.rst | 2 - ...-10-18-18-08-36.gh-issue-140301.m-2HxC.rst | 1 - ...-10-18-19-52-20.gh-issue-116738.NLJW0L.rst | 2 - ...-10-18-21-29-45.gh-issue-140306.xS5CcS.rst | 2 - ...-10-18-21-50-44.gh-issue-139109.9QQOzN.rst | 1 - ...-10-19-10-32-28.gh-issue-136895.HfsEh0.rst | 1 - ...-10-20-11-24-36.gh-issue-140358.UQuKdV.rst | 4 - ...-10-21-06-51-50.gh-issue-140406.0gJs8M.rst | 2 - ...-10-21-09-20-03.gh-issue-140398.SoABwJ.rst | 4 - ...-10-22-11-30-16.gh-issue-135904.3WE5oW.rst | 3 - ...-10-22-12-48-05.gh-issue-140476.F3-d1P.rst | 2 - ...-10-22-17-22-22.gh-issue-140431.m8D_A-.rst | 3 - ...-10-22-23-26-37.gh-issue-140443.wT5i1A.rst | 5 - ...-10-23-16-05-50.gh-issue-140471.Ax_aXn.rst | 2 - ...-10-24-14-29-12.gh-issue-133467.A5d6TM.rst | 1 - ...-10-24-20-16-42.gh-issue-140517.cqun-K.rst | 3 - ...-10-24-20-42-33.gh-issue-140551.-9swrl.rst | 2 - ...-10-25-07-25-52.gh-issue-140544.lwjtQe.rst | 1 - ...-10-25-17-36-46.gh-issue-140576.kj0SCY.rst | 2 - ...-10-25-21-31-43.gh-issue-131527.V-JVNP.rst | 2 - ...-10-29-11-31-59.gh-issue-140729.t9JsNt.rst | 2 - ...-10-29-20-59-10.gh-issue-140373.-uoaPP.rst | 2 - ...5-10-31-14-03-42.gh-issue-90344.gvZigO.rst | 1 - ...-11-02-12-47-38.gh-issue-140530.S934bp.rst | 2 - ...-11-02-15-28-33.gh-issue-140260.JNzlGz.rst | 2 - ...-11-03-17-21-38.gh-issue-140939.FVboAw.rst | 2 - ...-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst | 1 - ...-11-04-12-18-06.gh-issue-140942.GYns6n.rst | 2 - ...-11-05-19-50-37.gh-issue-140643.QCEOqG.rst | 3 - ...-11-10-23-07-06.gh-issue-141312.H-58GB.rst | 2 - ...-11-11-13-40-45.gh-issue-141367.I5KY7F.rst | 2 - ...-11-14-00-19-45.gh-issue-141528.VWdax1.rst | 3 - ...-11-14-16-25-15.gh-issue-114203.n3tlQO.rst | 1 - ...-11-15-01-21-00.gh-issue-141579.aB7cD9.rst | 2 - ...9-06-02-13-56-16.gh-issue-81313.axawSH.rst | 1 - ...-03-21-10-59-40.gh-issue-102431.eUDnf4.rst | 2 - ...-05-28-17-14-30.gh-issue-119668.RrIGpn.rst | 1 - ...-06-26-16-16-43.gh-issue-121011.qW54eh.rst | 2 - ...-08-08-12-39-36.gh-issue-122255.J_gU8Y.rst | 4 - ...-03-04-17-19-26.gh-issue-130693.Kv01r8.rst | 1 - ...-03-12-18-57-10.gh-issue-131116.uTpwXZ.rst | 2 - ...-04-18-18-08-05.gh-issue-132686.6kV_Gs.rst | 2 - ...-05-07-22-09-28.gh-issue-133601.9kUL3P.rst | 1 - ...-05-10-15-10-54.gh-issue-133789.I-ZlUX.rst | 1 - ...-06-10-18-02-29.gh-issue-135307.fXGrcK.rst | 2 - ...-06-29-22-01-00.gh-issue-133390.I1DW_3.rst | 2 - ...-07-01-04-57-57.gh-issue-136057.4-t596.rst | 1 - ...5-07-14-09-33-17.gh-issue-55531.Gt2e12.rst | 4 - ...-08-11-04-52-18.gh-issue-137627.Ku5Yi2.rst | 1 - ...5-08-15-20-35-30.gh-issue-69528.qc-Eh_.rst | 2 - ...-08-26-08-17-56.gh-issue-138151.I6CdAk.rst | 3 - ...-09-03-18-26-07.gh-issue-138425.cVE9Ho.rst | 2 - ...5-09-03-20-18-39.gh-issue-98896.tjez89.rst | 2 - ...-09-11-15-03-37.gh-issue-138775.w7rnSx.rst | 2 - ...-09-12-09-34-37.gh-issue-138764.mokHoY.rst | 3 - ...-09-13-12-19-17.gh-issue-138859.PxjIoN.rst | 1 - ...-09-15-21-03-11.gh-issue-138891.oZFdtR.rst | 2 - ...5-09-18-21-25-41.gh-issue-83714.TQjDWZ.rst | 2 - ...-09-23-09-46-46.gh-issue-139246.pzfM-w.rst | 1 - ...-09-25-20-16-10.gh-issue-101828.yTxJlJ.rst | 3 - ...5-09-30-12-52-54.gh-issue-63161.mECM1A.rst | 3 - ...-10-02-22-29-00.gh-issue-139462.VZXUHe.rst | 3 - ...-10-11-09-07-06.gh-issue-139940.g54efZ.rst | 1 - ...-10-13-11-25-41.gh-issue-136702.uvLGK1.rst | 3 - ...5-10-14-20-27-06.gh-issue-76007.2NcUbo.rst | 2 - ...-10-15-02-26-50.gh-issue-140135.54JYfM.rst | 2 - ...-10-15-15-10-34.gh-issue-140166.NtxRez.rst | 1 - ...-10-15-17-23-51.gh-issue-140141.j2mUDB.rst | 5 - ...-10-15-20-47-04.gh-issue-140120.3gffZq.rst | 2 - ...-10-15-21-42-13.gh-issue-140041._Fka2j.rst | 1 - ...-10-16-16-10-11.gh-issue-139707.zR6Qtn.rst | 2 - ...-10-16-17-17-20.gh-issue-135801.faH3fa.rst | 6 - ...-10-16-22-49-16.gh-issue-140212.llBNd0.rst | 5 - ...-10-17-12-33-01.gh-issue-140251.esM-OX.rst | 1 - ...-10-17-20-42-38.gh-issue-129117.X9jr4p.rst | 3 - ...-10-17-23-58-11.gh-issue-140272.lhY8uS.rst | 1 - ...5-10-18-14-30-21.gh-issue-76007.peEgcr.rst | 1 - ...5-10-18-15-20-25.gh-issue-76007.SNUzRq.rst | 2 - ...-10-20-12-33-49.gh-issue-140348.SAKnQZ.rst | 3 - ...-10-21-15-54-13.gh-issue-137530.ZyIVUH.rst | 1 - ...-10-22-12-56-57.gh-issue-140448.GsEkXD.rst | 2 - ...-10-22-20-52-13.gh-issue-140474.xIWlip.rst | 2 - ...-10-23-12-12-22.gh-issue-138774.mnh2gU.rst | 2 - ...-10-23-13-42-15.gh-issue-140481.XKxWpq.rst | 1 - ...-10-23-19-39-16.gh-issue-138162.Znw5DN.rst | 2 - ...-10-25-21-04-00.gh-issue-140607.oOZGxS.rst | 2 - ...-10-25-21-26-16.gh-issue-140593.OxlLc9.rst | 3 - ...-10-25-22-55-07.gh-issue-140601.In3MlS.rst | 4 - ...-10-26-16-24-12.gh-issue-140633.ioayC1.rst | 2 - ...-10-27-00-40-49.gh-issue-140650.DYJPJ9.rst | 3 - ...-10-27-13-49-31.gh-issue-140634.ULng9G.rst | 1 - ...-10-27-16-01-41.gh-issue-125434.qy0uRA.rst | 2 - ...-10-27-18-29-42.gh-issue-140590.LT9HHn.rst | 2 - ...-10-28-02-46-56.gh-issue-139946.aN3_uY.rst | 1 - ...-10-28-17-43-51.gh-issue-140228.8kfHhO.rst | 1 - ...-10-29-09-40-10.gh-issue-140741.L13UCV.rst | 2 - ...-10-29-16-12-41.gh-issue-120057.qGj5Dl.rst | 1 - ...-10-29-16-53-00.gh-issue-140766.CNagKF.rst | 1 - ...-10-30-12-36-19.gh-issue-140790._3T6-N.rst | 1 - ...-10-30-15-33-07.gh-issue-137821.8_Iavt.rst | 2 - ...-10-31-13-57-55.gh-issue-103847.VM7TnW.rst | 1 - ...-10-31-15-06-26.gh-issue-140691.JzHGtg.rst | 3 - ...-10-31-16-25-13.gh-issue-140808.XBiQ4j.rst | 1 - ...-11-01-00-34-53.gh-issue-140826.JEDd7U.rst | 2 - ...-11-01-00-36-14.gh-issue-140874.eAWt3K.rst | 1 - ...-11-01-14-44-09.gh-issue-140873.kfuc9B.rst | 2 - ...-11-02-09-37-22.gh-issue-140734.f8gST9.rst | 2 - ...-11-02-11-46-00.gh-issue-100218.9Ezfdq.rst | 3 - ...-11-02-19-23-32.gh-issue-140815.McEG-T.rst | 2 - ...-11-03-05-38-31.gh-issue-125115.jGS8MN.rst | 1 - ...-11-03-16-23-54.gh-issue-140797.DuFEeR.rst | 2 - ...5-11-04-12-16-13.gh-issue-75593.EFVhKR.rst | 1 - ...-11-04-15-40-35.gh-issue-137969.9VZQVt.rst | 3 - ...-11-04-20-08-41.gh-issue-141018.d_oyOI.rst | 2 - ...-11-06-15-11-50.gh-issue-141141.tgIfgH.rst | 1 - ...5-11-07-12-25-46.gh-issue-85524.9SWFIC.rst | 3 - ...5-11-08-13-03-10.gh-issue-87710.XJeZlP.rst | 1 - ...-11-09-18-55-13.gh-issue-141311.qZ3swc.rst | 2 - ...-11-10-01-47-18.gh-issue-141314.baaa28.rst | 1 - ...-11-12-01-49-03.gh-issue-137109.D6sq2B.rst | 5 - ...-11-12-15-42-47.gh-issue-124111.hTw4OE.rst | 2 - ...-11-13-14-51-30.gh-issue-140938.kXsHHv.rst | 2 - ...-11-14-16-24-20.gh-issue-141497.L_CxDJ.rst | 4 - ...-05-30-22-33-27.gh-issue-136065.bu337o.rst | 1 - ...-06-28-13-23-53.gh-issue-136063.aGk0Jv.rst | 2 - ...-08-15-23-08-44.gh-issue-137836.b55rhh.rst | 3 - ...-07-09-21-45-51.gh-issue-136442.jlbklP.rst | 1 - ...-10-15-00-52-12.gh-issue-140082.fpET50.rst | 3 - ...-10-23-16-39-49.gh-issue-140482.ZMtyeD.rst | 1 - ...-09-20-20-31-54.gh-issue-139188.zfcxkW.rst | 1 - ...-09-21-10-30-08.gh-issue-139198.Fm7NfU.rst | 1 - ...-10-29-15-20-19.gh-issue-140702.ZXtW8h.rst | 2 - ...-11-12-12-54-28.gh-issue-141442.50dS3P.rst | 1 - ...-11-04-19-20-05.gh-issue-140849.YjB2ZZ.rst | 1 - README.rst | 2 +- 192 files changed, 1811 insertions(+), 394 deletions(-) create mode 100644 Misc/NEWS.d/3.15.0a2.rst delete mode 100644 Misc/NEWS.d/next/Build/2025-08-10-22-28-06.gh-issue-137618.FdNvIE.rst delete mode 100644 Misc/NEWS.d/next/Build/2025-10-16-11-30-53.gh-issue-140189.YCrUyt.rst delete mode 100644 Misc/NEWS.d/next/Build/2025-10-17-11-33-45.gh-issue-140239._k-GgW.rst delete mode 100644 Misc/NEWS.d/next/Build/2025-10-22-12-44-07.gh-issue-140475.OhzQbR.rst delete mode 100644 Misc/NEWS.d/next/Build/2025-10-25-08-07-06.gh-issue-140513.6OhLTs.rst delete mode 100644 Misc/NEWS.d/next/Build/2025-10-29-12-30-38.gh-issue-140768.ITYrzw.rst delete mode 100644 Misc/NEWS.d/next/Build/2025-10-31-13-20-16.gh-issue-140454.gF6dCe.rst delete mode 100644 Misc/NEWS.d/next/C_API/2025-10-06-22-17-47.gh-issue-139653.6-1MOd.rst delete mode 100644 Misc/NEWS.d/next/C_API/2025-10-15-15-59-59.gh-issue-140153.BO7sH4.rst delete mode 100644 Misc/NEWS.d/next/C_API/2025-10-26-16-45-06.gh-issue-140487.fGOqss.rst delete mode 100644 Misc/NEWS.d/next/C_API/2025-10-26-16-45-28.gh-issue-140556.s__Dae.rst delete mode 100644 Misc/NEWS.d/next/C_API/2025-11-05-04-38-16.gh-issue-141004.rJL43P.rst delete mode 100644 Misc/NEWS.d/next/C_API/2025-11-05-05-45-49.gh-issue-141004.N9Ooh9.rst delete mode 100644 Misc/NEWS.d/next/C_API/2025-11-06-06-28-14.gh-issue-141042.brOioJ.rst delete mode 100644 Misc/NEWS.d/next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst delete mode 100644 Misc/NEWS.d/next/C_API/2025-11-10-11-26-26.gh-issue-141341.OsO6-y.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-06-24-13-12-58.gh-issue-134786.MF0VVk.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-00-41-46.gh-issue-136327.7AiTb_.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-07-29-17-51-14.gh-issue-131253.GpRjWy.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-09-13-01-23-25.gh-issue-138857.YQ5gdc.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-09-15-13-06-11.gh-issue-138944.PeCgLb.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-09-23-21-01-12.gh-issue-139269.1rIaxy.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-03-17-51-43.gh-issue-139475._684ED.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-10-03-37.gh-issue-139640.gY5oTb.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-10-03-37.gh-issue-139640.gY5oTb2.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-14-19-47.gh-issue-135801.OhxEZS.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-12-01-12-12.gh-issue-139817.PAn-8Z.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-13-13-54-19.gh-issue-139914.M-y_3E.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-17-07-37.gh-issue-140067.ID2gOm.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-18-24-16.gh-issue-139871.SWtuUz.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-20-18-31.gh-issue-140080.8ROjxW.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-15-00-21-40.gh-issue-140061.J0XeDV.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-15-17-12-32.gh-issue-140149.cy1m3d.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-16-21-47-00.gh-issue-140104.A8SQIm.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-14-38-10.gh-issue-140253.gCqFaL.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-18-03-12.gh-issue-139951.IdwM2O.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-20-23-19.gh-issue-140257.8Txmem.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-18-08-36.gh-issue-140301.m-2HxC.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-19-52-20.gh-issue-116738.NLJW0L.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-29-45.gh-issue-140306.xS5CcS.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-19-10-32-28.gh-issue-136895.HfsEh0.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-20-11-24-36.gh-issue-140358.UQuKdV.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-21-06-51-50.gh-issue-140406.0gJs8M.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-21-09-20-03.gh-issue-140398.SoABwJ.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-11-30-16.gh-issue-135904.3WE5oW.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-12-48-05.gh-issue-140476.F3-d1P.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-17-22-22.gh-issue-140431.m8D_A-.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-23-26-37.gh-issue-140443.wT5i1A.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-23-16-05-50.gh-issue-140471.Ax_aXn.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-14-29-12.gh-issue-133467.A5d6TM.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-16-42.gh-issue-140517.cqun-K.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-42-33.gh-issue-140551.-9swrl.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-07-25-52.gh-issue-140544.lwjtQe.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-21-31-43.gh-issue-131527.V-JVNP.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-11-31-59.gh-issue-140729.t9JsNt.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-20-59-10.gh-issue-140373.-uoaPP.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-31-14-03-42.gh-issue-90344.gvZigO.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-12-47-38.gh-issue-140530.S934bp.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-15-28-33.gh-issue-140260.JNzlGz.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-03-17-21-38.gh-issue-140939.FVboAw.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-12-18-06.gh-issue-140942.GYns6n.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-05-19-50-37.gh-issue-140643.QCEOqG.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-10-23-07-06.gh-issue-141312.H-58GB.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-11-13-40-45.gh-issue-141367.I5KY7F.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-00-19-45.gh-issue-141528.VWdax1.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-16-25-15.gh-issue-114203.n3tlQO.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-01-21-00.gh-issue-141579.aB7cD9.rst delete mode 100644 Misc/NEWS.d/next/Library/2019-06-02-13-56-16.gh-issue-81313.axawSH.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-21-10-59-40.gh-issue-102431.eUDnf4.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-05-28-17-14-30.gh-issue-119668.RrIGpn.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-06-26-16-16-43.gh-issue-121011.qW54eh.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-08-08-12-39-36.gh-issue-122255.J_gU8Y.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-03-04-17-19-26.gh-issue-130693.Kv01r8.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-03-12-18-57-10.gh-issue-131116.uTpwXZ.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-04-18-18-08-05.gh-issue-132686.6kV_Gs.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-07-22-09-28.gh-issue-133601.9kUL3P.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-10-15-10-54.gh-issue-133789.I-ZlUX.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-06-10-18-02-29.gh-issue-135307.fXGrcK.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-06-29-22-01-00.gh-issue-133390.I1DW_3.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-07-01-04-57-57.gh-issue-136057.4-t596.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-07-14-09-33-17.gh-issue-55531.Gt2e12.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-08-11-04-52-18.gh-issue-137627.Ku5Yi2.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-08-15-20-35-30.gh-issue-69528.qc-Eh_.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-08-26-08-17-56.gh-issue-138151.I6CdAk.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-09-03-18-26-07.gh-issue-138425.cVE9Ho.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-09-03-20-18-39.gh-issue-98896.tjez89.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-09-11-15-03-37.gh-issue-138775.w7rnSx.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-09-12-09-34-37.gh-issue-138764.mokHoY.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-09-13-12-19-17.gh-issue-138859.PxjIoN.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-09-15-21-03-11.gh-issue-138891.oZFdtR.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-09-18-21-25-41.gh-issue-83714.TQjDWZ.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-09-23-09-46-46.gh-issue-139246.pzfM-w.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-09-25-20-16-10.gh-issue-101828.yTxJlJ.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-09-30-12-52-54.gh-issue-63161.mECM1A.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-02-22-29-00.gh-issue-139462.VZXUHe.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-11-09-07-06.gh-issue-139940.g54efZ.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-13-11-25-41.gh-issue-136702.uvLGK1.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-14-20-27-06.gh-issue-76007.2NcUbo.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-15-02-26-50.gh-issue-140135.54JYfM.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-15-15-10-34.gh-issue-140166.NtxRez.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-15-17-23-51.gh-issue-140141.j2mUDB.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-15-20-47-04.gh-issue-140120.3gffZq.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-15-21-42-13.gh-issue-140041._Fka2j.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-16-16-10-11.gh-issue-139707.zR6Qtn.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-16-17-17-20.gh-issue-135801.faH3fa.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-16-22-49-16.gh-issue-140212.llBNd0.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-17-12-33-01.gh-issue-140251.esM-OX.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-17-20-42-38.gh-issue-129117.X9jr4p.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-17-23-58-11.gh-issue-140272.lhY8uS.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-18-14-30-21.gh-issue-76007.peEgcr.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-18-15-20-25.gh-issue-76007.SNUzRq.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-20-12-33-49.gh-issue-140348.SAKnQZ.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-21-15-54-13.gh-issue-137530.ZyIVUH.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-22-12-56-57.gh-issue-140448.GsEkXD.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-22-20-52-13.gh-issue-140474.xIWlip.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-23-12-12-22.gh-issue-138774.mnh2gU.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-23-13-42-15.gh-issue-140481.XKxWpq.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-23-19-39-16.gh-issue-138162.Znw5DN.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-25-21-04-00.gh-issue-140607.oOZGxS.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-25-21-26-16.gh-issue-140593.OxlLc9.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-25-22-55-07.gh-issue-140601.In3MlS.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-26-16-24-12.gh-issue-140633.ioayC1.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-27-00-40-49.gh-issue-140650.DYJPJ9.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-27-13-49-31.gh-issue-140634.ULng9G.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-27-16-01-41.gh-issue-125434.qy0uRA.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-27-18-29-42.gh-issue-140590.LT9HHn.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-28-02-46-56.gh-issue-139946.aN3_uY.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-28-17-43-51.gh-issue-140228.8kfHhO.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-29-09-40-10.gh-issue-140741.L13UCV.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-29-16-12-41.gh-issue-120057.qGj5Dl.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-29-16-53-00.gh-issue-140766.CNagKF.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-30-12-36-19.gh-issue-140790._3T6-N.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-30-15-33-07.gh-issue-137821.8_Iavt.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-31-13-57-55.gh-issue-103847.VM7TnW.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-31-15-06-26.gh-issue-140691.JzHGtg.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-10-31-16-25-13.gh-issue-140808.XBiQ4j.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-01-00-34-53.gh-issue-140826.JEDd7U.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-01-00-36-14.gh-issue-140874.eAWt3K.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-01-14-44-09.gh-issue-140873.kfuc9B.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-02-09-37-22.gh-issue-140734.f8gST9.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-02-11-46-00.gh-issue-100218.9Ezfdq.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-02-19-23-32.gh-issue-140815.McEG-T.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-03-05-38-31.gh-issue-125115.jGS8MN.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-03-16-23-54.gh-issue-140797.DuFEeR.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-04-12-16-13.gh-issue-75593.EFVhKR.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-04-15-40-35.gh-issue-137969.9VZQVt.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-04-20-08-41.gh-issue-141018.d_oyOI.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-06-15-11-50.gh-issue-141141.tgIfgH.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-07-12-25-46.gh-issue-85524.9SWFIC.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-08-13-03-10.gh-issue-87710.XJeZlP.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-09-18-55-13.gh-issue-141311.qZ3swc.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-10-01-47-18.gh-issue-141314.baaa28.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-12-01-49-03.gh-issue-137109.D6sq2B.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-12-15-42-47.gh-issue-124111.hTw4OE.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-13-14-51-30.gh-issue-140938.kXsHHv.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-11-14-16-24-20.gh-issue-141497.L_CxDJ.rst delete mode 100644 Misc/NEWS.d/next/Security/2025-05-30-22-33-27.gh-issue-136065.bu337o.rst delete mode 100644 Misc/NEWS.d/next/Security/2025-06-28-13-23-53.gh-issue-136063.aGk0Jv.rst delete mode 100644 Misc/NEWS.d/next/Security/2025-08-15-23-08-44.gh-issue-137836.b55rhh.rst delete mode 100644 Misc/NEWS.d/next/Tests/2025-07-09-21-45-51.gh-issue-136442.jlbklP.rst delete mode 100644 Misc/NEWS.d/next/Tests/2025-10-15-00-52-12.gh-issue-140082.fpET50.rst delete mode 100644 Misc/NEWS.d/next/Tests/2025-10-23-16-39-49.gh-issue-140482.ZMtyeD.rst delete mode 100644 Misc/NEWS.d/next/Tools-Demos/2025-09-20-20-31-54.gh-issue-139188.zfcxkW.rst delete mode 100644 Misc/NEWS.d/next/Tools-Demos/2025-09-21-10-30-08.gh-issue-139198.Fm7NfU.rst delete mode 100644 Misc/NEWS.d/next/Tools-Demos/2025-10-29-15-20-19.gh-issue-140702.ZXtW8h.rst delete mode 100644 Misc/NEWS.d/next/Tools-Demos/2025-11-12-12-54-28.gh-issue-141442.50dS3P.rst delete mode 100644 Misc/NEWS.d/next/Windows/2025-11-04-19-20-05.gh-issue-140849.YjB2ZZ.rst diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 24e673d3d1394f..971c6a69e5d065 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -353,4 +353,4 @@ Importing Modules On error, return NULL with an exception set. - .. versionadded:: next + .. versionadded:: 3.15 diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 18ee16118070eb..3cac2c8b213c80 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -1390,7 +1390,7 @@ All of the following functions must be called after :c:func:`Py_Initialize`. See :c:func:`PyUnstable_ThreadState_ResetStackProtection` for undoing this operation. - .. versionadded:: next + .. versionadded:: 3.15 .. c:function:: void PyUnstable_ThreadState_ResetStackProtection(PyThreadState *tstate) @@ -1400,7 +1400,7 @@ All of the following functions must be called after :c:func:`Py_Initialize`. See :c:func:`PyUnstable_ThreadState_SetStackProtection` for an explanation. - .. versionadded:: next + .. versionadded:: 3.15 .. c:function:: PyInterpreterState* PyInterpreterState_Get(void) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 0ea3c3c59a660d..2e7d0dbc26e5bc 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -2261,7 +2261,7 @@ and classes for traversing abstract syntax trees: The minimum supported version for ``feature_version`` is now ``(3, 7)``. The ``optimize`` argument was added. - .. versionadded:: next + .. versionadded:: 3.15 Added the *module* parameter. diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst index ba882f10bbe2b8..059377756999a4 100644 --- a/Doc/library/decimal.rst +++ b/Doc/library/decimal.rst @@ -1575,7 +1575,7 @@ Constants Specification that this implementation complies with. See https://speleotrove.com/decimal/decarith.html for the specification. - .. versionadded:: next + .. versionadded:: 3.15 The following constants are only relevant for the C module. They diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 3257daf89d327b..8314fed80fa512 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -377,7 +377,7 @@ are always available. They are listed here in alphabetical order. ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` can now be passed in flags to enable support for top-level ``await``, ``async for``, and ``async with``. - .. versionadded:: next + .. versionadded:: 3.15 Added the *module* parameter. diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index b2e2e11c0dc414..97136b234084fc 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -716,7 +716,7 @@ The :mod:`functools` module defines the following functions: .. versionadded:: 3.8 - .. versionchanged:: next + .. versionchanged:: 3.15 Added support of non-:term:`descriptor` callables. diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 03ba23b6216cbf..3f0a54ac535cd6 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -480,7 +480,7 @@ ABC hierarchy:: .. versionchanged:: 3.5 Made the method static. - .. versionadded:: next + .. versionadded:: 3.15 Added the *fullname* parameter. @@ -1048,7 +1048,7 @@ find and load modules. :meth:`PathFinder.invalidate_caches` invalidates :class:`NamespacePath`, forcing the path value to be recomputed next time it is accessed. - .. versionadded:: next + .. versionadded:: 3.15 .. class:: SourceFileLoader(fullname, path) diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index c00db31a8ec051..5220c559d3d857 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -636,7 +636,7 @@ Retrieving source code .. versionchanged:: 3.5 Documentation strings are now inherited if not overridden. - .. versionchanged:: next + .. versionchanged:: 3.15 Added parameters *inherit_class_doc* and *fallback_to_class_doc*. Documentation strings on :class:`~functools.cached_property` diff --git a/Doc/library/math.integer.rst b/Doc/library/math.integer.rst index 6a9fe74c5e861b..0068ae2bdd5d07 100644 --- a/Doc/library/math.integer.rst +++ b/Doc/library/math.integer.rst @@ -4,7 +4,7 @@ .. module:: math.integer :synopsis: Integer-specific mathematics functions. -.. versionadded:: next +.. versionadded:: 3.15 -------------- diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 54c98346b2798b..186f99e9591546 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -781,7 +781,7 @@ the following functions from the :mod:`math.integer` module: Floats with integral values (like ``5.0``) are no longer accepted in the :func:`factorial` function. -.. deprecated:: next +.. deprecated:: 3.15 These aliases are :term:`soft deprecated` in favor of the :mod:`math.integer` functions. diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 7dc6c177268ec2..671270d6112212 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -3404,7 +3404,7 @@ features: .. availability:: Linux >= 4.11 with glibc >= 2.28. - .. versionadded:: next + .. versionadded:: 3.15 .. class:: statx_result @@ -3661,7 +3661,7 @@ features: .. availability:: Linux >= 4.11 with glibc >= 2.28. - .. versionadded:: next + .. versionadded:: 3.15 .. data:: STATX_TYPE @@ -3690,7 +3690,7 @@ features: .. availability:: Linux >= 4.11 with glibc >= 2.28. - .. versionadded:: next + .. versionadded:: 3.15 .. data:: AT_STATX_FORCE_SYNC @@ -3700,7 +3700,7 @@ features: .. availability:: Linux >= 4.11 with glibc >= 2.28. - .. versionadded:: next + .. versionadded:: 3.15 .. data:: AT_STATX_DONT_SYNC @@ -3709,7 +3709,7 @@ features: .. availability:: Linux >= 4.11 with glibc >= 2.28. - .. versionadded:: next + .. versionadded:: 3.15 .. data:: AT_STATX_SYNC_AS_STAT @@ -3721,7 +3721,7 @@ features: .. availability:: Linux >= 4.11 with glibc >= 2.28. - .. versionadded:: next + .. versionadded:: 3.15 .. data:: AT_NO_AUTOMOUNT @@ -3733,7 +3733,7 @@ features: .. availability:: Linux. - .. versionadded:: next + .. versionadded:: 3.15 .. function:: statvfs(path) diff --git a/Doc/library/stat.rst b/Doc/library/stat.rst index 1cbec3ab847c5f..82012b31a00f20 100644 --- a/Doc/library/stat.rst +++ b/Doc/library/stat.rst @@ -511,4 +511,4 @@ meaning of these constants. STATX_ATTR_DAX STATX_ATTR_WRITE_ATOMIC - .. versionadded:: next + .. versionadded:: 3.15 diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index c539345e598777..3bcaba0b3e1eba 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -3191,7 +3191,7 @@ objects. Taking all bytes is a zero-copy operation. - .. versionadded:: next + .. versionadded:: 3.15 See the :ref:`What's New ` entry for common code patterns which can be optimized with diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst index c0d9e79197de7c..f5e6f9f8acfdb8 100644 --- a/Doc/library/symtable.rst +++ b/Doc/library/symtable.rst @@ -30,7 +30,7 @@ Generating Symbol Tables It is needed to unambiguous :ref:`filter ` syntax warnings by module name. - .. versionadded:: next + .. versionadded:: 3.15 Added the *module* parameter. diff --git a/Doc/library/unicodedata.rst b/Doc/library/unicodedata.rst index fd5f56bd7eaaeb..34f21f49b4bcb1 100644 --- a/Doc/library/unicodedata.rst +++ b/Doc/library/unicodedata.rst @@ -156,7 +156,7 @@ following functions: >>> unicodedata.isxidstart('0') False - .. versionadded:: next + .. versionadded:: 3.15 .. function:: isxidcontinue(chr, /) @@ -171,7 +171,7 @@ following functions: >>> unicodedata.isxidcontinue(' ') False - .. versionadded:: next + .. versionadded:: 3.15 .. function:: decomposition(chr, /) diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst index 2f3cf6008f58e2..0de7a90bfcb60e 100644 --- a/Doc/library/warnings.rst +++ b/Doc/library/warnings.rst @@ -513,7 +513,7 @@ Available Functions .. versionchanged:: 3.6 Add the *source* parameter. - .. versionchanged:: next + .. versionchanged:: 3.15 If no module is passed, test the filter regular expression against module names created from the path, not only the path itself. diff --git a/Doc/library/winreg.rst b/Doc/library/winreg.rst index b150c53735d634..89def6e2afe088 100644 --- a/Doc/library/winreg.rst +++ b/Doc/library/winreg.rst @@ -818,6 +818,6 @@ integer handle, and also disconnect the Windows handle from the handle object. will automatically close *key* when control leaves the :keyword:`with` block. -.. versionchanged:: next +.. versionchanged:: 3.15 Handle objects are now compared by their underlying Windows handle value instead of object identity for equality comparisons. diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index cbbc87b4721a9f..e59759683a6d4c 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -656,7 +656,7 @@ Functions .. versionchanged:: 3.13 Added the :meth:`!close` method. - .. versionchanged:: next + .. versionchanged:: 3.15 A :exc:`ResourceWarning` is now emitted if the iterator opened a file and is not explicitly closed. diff --git a/Include/patchlevel.h b/Include/patchlevel.h index e3996ee86793dd..899c892631fafa 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -24,10 +24,10 @@ #define PY_MINOR_VERSION 15 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA -#define PY_RELEASE_SERIAL 1 +#define PY_RELEASE_SERIAL 2 /* Version as a string */ -#define PY_VERSION "3.15.0a1+" +#define PY_VERSION "3.15.0a2" /*--end constants--*/ diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 293c3189589e36..11ffc6bf3a1bb5 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,4 +1,4 @@ -# Autogenerated by Sphinx on Tue Oct 14 13:46:01 2025 +# Autogenerated by Sphinx on Tue Nov 18 16:51:09 2025 # as part of the release process. topics = { @@ -1098,10 +1098,10 @@ class and instance attributes applies as for regular assignments. 'bltin-ellipsis-object': r'''The Ellipsis Object ******************* -This object is commonly used used to indicate that something is -omitted. It supports no special operations. There is exactly one -ellipsis object, named "Ellipsis" (a built-in name). -"type(Ellipsis)()" produces the "Ellipsis" singleton. +This object is commonly used to indicate that something is omitted. It +supports no special operations. There is exactly one ellipsis object, +named "Ellipsis" (a built-in name). "type(Ellipsis)()" produces the +"Ellipsis" singleton. It is written as "Ellipsis" or "...". @@ -4140,6 +4140,10 @@ def double(x): available for commands and command arguments, e.g. the current global and local names are offered as arguments of the "p" command. + +Command-line interface +====================== + You can also invoke "pdb" from the command line to debug other scripts. For example: @@ -4155,7 +4159,7 @@ def double(x): -c, --command To execute commands as if given in a ".pdbrc" file; see Debugger - Commands. + commands. Changed in version 3.2: Added the "-c" option. @@ -4376,7 +4380,7 @@ class pdb.Pdb(completekey='tab', stdin=None, stdout=None, skip=None, nosigint=Fa See the documentation for the functions explained above. -Debugger Commands +Debugger commands ================= The commands recognized by the debugger are listed below. Most @@ -5616,9 +5620,8 @@ class of the instance or a *non-virtual base class* thereof. The 2.71828 4.0 -Unlike in integer literals, leading zeros are allowed in the numeric -parts. For example, "077.010" is legal, and denotes the same number as -"77.10". +Unlike in integer literals, leading zeros are allowed. For example, +"077.010" is legal, and denotes the same number as "77.01". As in integer literals, single underscores may occur between digits to help readability: @@ -7435,9 +7438,8 @@ class body. A "SyntaxError" is raised if a variable is used or 2.71828 4.0 -Unlike in integer literals, leading zeros are allowed in the numeric -parts. For example, "077.010" is legal, and denotes the same number as -"77.10". +Unlike in integer literals, leading zeros are allowed. For example, +"077.010" is legal, and denotes the same number as "77.01". As in integer literals, single underscores may occur between digits to help readability: @@ -7685,9 +7687,8 @@ class that has an "__rsub__()" method, "type(y).__rsub__(y, x)" is ************************* *Objects* are Python’s abstraction for data. All data in a Python -program is represented by objects or by relations between objects. (In -a sense, and in conformance to Von Neumann’s model of a “stored -program computer”, code is also represented by objects.) +program is represented by objects or by relations between objects. +Even code is represented by objects. Every object has an identity, a type and a value. An object’s *identity* never changes once it has been created; you may think of it @@ -10301,6 +10302,17 @@ class is used in a class pattern with positional arguments, each follow uncased characters and lowercase characters only cased ones. Return "False" otherwise. + For example: + + >>> 'Spam, Spam, Spam'.istitle() + True + >>> 'spam, spam, spam'.istitle() + False + >>> 'SPAM, SPAM, SPAM'.istitle() + False + + See also "title()". + str.isupper() Return "True" if all cased characters [4] in the string are @@ -10663,6 +10675,8 @@ class is used in a class pattern with positional arguments, each >>> titlecase("they're bill's friends.") "They're Bill's Friends." + See also "istitle()". + str.translate(table, /) Return a copy of the string in which each character has been mapped @@ -12362,6 +12376,11 @@ class method object, it is transformed into an instance method object | | "X.__bases__" will be exactly equal to "(A, B, | | | C)". | +----------------------------------------------------+----------------------------------------------------+ +| type.__base__ | **CPython implementation detail:** The single base | +| | class in the inheritance chain that is responsible | +| | for the memory layout of instances. This attribute | +| | corresponds to "tp_base" at the C level. | ++----------------------------------------------------+----------------------------------------------------+ | type.__doc__ | The class’s documentation string, or "None" if | | | undefined. Not inherited by subclasses. | +----------------------------------------------------+----------------------------------------------------+ diff --git a/Misc/NEWS.d/3.15.0a2.rst b/Misc/NEWS.d/3.15.0a2.rst new file mode 100644 index 00000000000000..ba82c854fac2d4 --- /dev/null +++ b/Misc/NEWS.d/3.15.0a2.rst @@ -0,0 +1,1746 @@ +.. date: 2025-11-04-19-20-05 +.. gh-issue: 140849 +.. nonce: YjB2ZZ +.. release date: 2025-11-18 +.. section: Windows + +Update bundled liblzma to version 5.8.1. + +.. + +.. date: 2025-11-12-12-54-28 +.. gh-issue: 141442 +.. nonce: 50dS3P +.. section: Tools/Demos + +The iOS testbed now correctly handles test arguments that contain spaces. + +.. + +.. date: 2025-10-29-15-20-19 +.. gh-issue: 140702 +.. nonce: ZXtW8h +.. section: Tools/Demos + +The iOS testbed app will now expose the ``GITHUB_ACTIONS`` environment +variable to iOS apps being tested. + +.. + +.. date: 2025-09-21-10-30-08 +.. gh-issue: 139198 +.. nonce: Fm7NfU +.. section: Tools/Demos + +Remove ``Tools/scripts/checkpip.py`` script. + +.. + +.. date: 2025-09-20-20-31-54 +.. gh-issue: 139188 +.. nonce: zfcxkW +.. section: Tools/Demos + +Remove ``Tools/tz/zdump.py`` script. + +.. + +.. date: 2025-10-23-16-39-49 +.. gh-issue: 140482 +.. nonce: ZMtyeD +.. section: Tests + +Preserve and restore the state of ``stty echo`` as part of the test +environment. + +.. + +.. date: 2025-10-15-00-52-12 +.. gh-issue: 140082 +.. nonce: fpET50 +.. section: Tests + +Update ``python -m test`` to set ``FORCE_COLOR=1`` when being run with color +enabled so that :mod:`unittest` which is run by it with redirected output +will output in color. + +.. + +.. date: 2025-07-09-21-45-51 +.. gh-issue: 136442 +.. nonce: jlbklP +.. section: Tests + +Use exitcode ``1`` instead of ``5`` if :func:`unittest.TestCase.setUpClass` +raises an exception + +.. + +.. date: 2025-08-15-23-08-44 +.. gh-issue: 137836 +.. nonce: b55rhh +.. section: Security + +Add support of the "plaintext" element, RAWTEXT elements "xmp", "iframe", +"noembed" and "noframes", and optionally RAWTEXT element "noscript" in +:class:`html.parser.HTMLParser`. + +.. + +.. date: 2025-06-28-13-23-53 +.. gh-issue: 136063 +.. nonce: aGk0Jv +.. section: Security + +:mod:`email.message`: ensure linear complexity for legacy HTTP parameters +parsing. Patch by Bénédikt Tran. + +.. + +.. date: 2025-05-30-22-33-27 +.. gh-issue: 136065 +.. nonce: bu337o +.. section: Security + +Fix quadratic complexity in :func:`os.path.expandvars`. + +.. + +.. date: 2025-11-14-16-24-20 +.. gh-issue: 141497 +.. nonce: L_CxDJ +.. section: Library + +:mod:`ipaddress`: ensure that the methods :meth:`IPv4Network.hosts() +` and :meth:`IPv6Network.hosts() +` always return an iterator. + +.. + +.. date: 2025-11-13-14-51-30 +.. gh-issue: 140938 +.. nonce: kXsHHv +.. section: Library + +The :func:`statistics.stdev` and :func:`statistics.pstdev` functions now +raise a :exc:`ValueError` when the input contains an infinity or a NaN. + +.. + +.. date: 2025-11-12-15-42-47 +.. gh-issue: 124111 +.. nonce: hTw4OE +.. section: Library + +Updated Tcl threading configuration in :mod:`_tkinter` to assume that +threads are always available in Tcl 9 and later. + +.. + +.. date: 2025-11-12-01-49-03 +.. gh-issue: 137109 +.. nonce: D6sq2B +.. section: Library + +The :mod:`os.fork` and related forking APIs will no longer warn in the +common case where Linux or macOS platform APIs return the number of threads +in a process and find the answer to be 1 even when a +:func:`os.register_at_fork` ``after_in_parent=`` callback (re)starts a +thread. + +.. + +.. date: 2025-11-10-01-47-18 +.. gh-issue: 141314 +.. nonce: baaa28 +.. section: Library + +Fix assertion failure in :meth:`io.TextIOWrapper.tell` when reading files +with standalone carriage return (``\r``) line endings. + +.. + +.. date: 2025-11-09-18-55-13 +.. gh-issue: 141311 +.. nonce: qZ3swc +.. section: Library + +Fix assertion failure in :func:`!io.BytesIO.readinto` and undefined behavior +arising when read position is above capcity in :class:`io.BytesIO`. + +.. + +.. date: 2025-11-08-13-03-10 +.. gh-issue: 87710 +.. nonce: XJeZlP +.. section: Library + +:mod:`mimetypes`: Update mime type for ``.ai`` files to ``application/pdf``. + +.. + +.. date: 2025-11-07-12-25-46 +.. gh-issue: 85524 +.. nonce: 9SWFIC +.. section: Library + +Update ``io.FileIO.readall``, an implementation of +:meth:`io.RawIOBase.readall`, to follow :class:`io.IOBase` guidelines and +raise :exc:`io.UnsupportedOperation` when a file is in "w" mode rather than +:exc:`OSError` + +.. + +.. date: 2025-11-06-15-11-50 +.. gh-issue: 141141 +.. nonce: tgIfgH +.. section: Library + +Fix a thread safety issue with :func:`base64.b85decode`. Contributed by +Benel Tayar. + +.. + +.. date: 2025-11-04-20-08-41 +.. gh-issue: 141018 +.. nonce: d_oyOI +.. section: Library + +:mod:`mimetypes`: Update ``.exe``, ``.dll``, ``.rtf`` and (when +``strict=False``) ``.jpg`` to their correct IANA mime type. + +.. + +.. date: 2025-11-04-15-40-35 +.. gh-issue: 137969 +.. nonce: 9VZQVt +.. section: Library + +Fix :meth:`annotationlib.ForwardRef.evaluate` returning +:class:`~annotationlib.ForwardRef` objects which don't update with new +globals. + +.. + +.. date: 2025-11-04-12-16-13 +.. gh-issue: 75593 +.. nonce: EFVhKR +.. section: Library + +Add support of :term:`path-like objects ` and +:term:`bytes-like objects ` in :func:`wave.open`. + +.. + +.. date: 2025-11-03-16-23-54 +.. gh-issue: 140797 +.. nonce: DuFEeR +.. section: Library + +The undocumented :class:`!re.Scanner` class now forbids regular expressions +containing capturing groups in its lexicon patterns. Patterns using +capturing groups could previously lead to crashes with segmentation fault. +Use non-capturing groups (?:...) instead. + +.. + +.. date: 2025-11-03-05-38-31 +.. gh-issue: 125115 +.. nonce: jGS8MN +.. section: Library + +Refactor the :mod:`pdb` parsing issue so positional arguments can pass +through intuitively. + +.. + +.. date: 2025-11-02-19-23-32 +.. gh-issue: 140815 +.. nonce: McEG-T +.. section: Library + +:mod:`faulthandler` now detects if a frame or a code object is invalid or +freed. Patch by Victor Stinner. + +.. + +.. date: 2025-11-02-11-46-00 +.. gh-issue: 100218 +.. nonce: 9Ezfdq +.. section: Library + +Correctly set :attr:`~OSError.errno` when :func:`socket.if_nametoindex` or +:func:`socket.if_indextoname` raise an :exc:`OSError`. Patch by Bénédikt +Tran. + +.. + +.. date: 2025-11-02-09-37-22 +.. gh-issue: 140734 +.. nonce: f8gST9 +.. section: Library + +:mod:`multiprocessing`: fix off-by-one error when checking the length of a +temporary socket file path. Patch by Bénédikt Tran. + +.. + +.. date: 2025-11-01-14-44-09 +.. gh-issue: 140873 +.. nonce: kfuc9B +.. section: Library + +Add support of non-:term:`descriptor` callables in +:func:`functools.singledispatchmethod`. + +.. + +.. date: 2025-11-01-00-36-14 +.. gh-issue: 140874 +.. nonce: eAWt3K +.. section: Library + +Bump the version of pip bundled in ensurepip to version 25.3 + +.. + +.. date: 2025-11-01-00-34-53 +.. gh-issue: 140826 +.. nonce: JEDd7U +.. section: Library + +Now :class:`!winreg.HKEYType` objects are compared by their underlying +Windows registry handle value instead of their object identity. + +.. + +.. date: 2025-10-31-16-25-13 +.. gh-issue: 140808 +.. nonce: XBiQ4j +.. section: Library + +The internal class ``mailbox._ProxyFile`` is no longer a parameterized +generic. + +.. + +.. date: 2025-10-31-15-06-26 +.. gh-issue: 140691 +.. nonce: JzHGtg +.. section: Library + +In :mod:`urllib.request`, when opening a FTP URL fails because a data +connection cannot be made, the control connection's socket is now closed to +avoid a :exc:`ResourceWarning`. + +.. + +.. date: 2025-10-31-13-57-55 +.. gh-issue: 103847 +.. nonce: VM7TnW +.. section: Library + +Fix hang when cancelling process created by +:func:`asyncio.create_subprocess_exec` or +:func:`asyncio.create_subprocess_shell`. Patch by Kumar Aditya. + +.. + +.. date: 2025-10-30-15-33-07 +.. gh-issue: 137821 +.. nonce: 8_Iavt +.. section: Library + +Convert ``_json`` module to use Argument Clinic. Patched by Yoonho Hann. + +.. + +.. date: 2025-10-30-12-36-19 +.. gh-issue: 140790 +.. nonce: _3T6-N +.. section: Library + +Initialize all Pdb's instance variables in ``__init__``, remove some +hasattr/getattr + +.. + +.. date: 2025-10-29-16-53-00 +.. gh-issue: 140766 +.. nonce: CNagKF +.. section: Library + +Add :func:`enum.show_flag_values` and ``enum.bin`` to ``enum.__all__``. + +.. + +.. date: 2025-10-29-16-12-41 +.. gh-issue: 120057 +.. nonce: qGj5Dl +.. section: Library + +Add :func:`os.reload_environ` to ``os.__all__``. + +.. + +.. date: 2025-10-29-09-40-10 +.. gh-issue: 140741 +.. nonce: L13UCV +.. section: Library + +Fix :func:`profiling.sampling.sample` incorrectly handling a +:exc:`FileNotFoundError` or :exc:`PermissionError`. + +.. + +.. date: 2025-10-28-17-43-51 +.. gh-issue: 140228 +.. nonce: 8kfHhO +.. section: Library + +Avoid making unnecessary filesystem calls for frozen modules in +:mod:`linecache` when the global module cache is not present. + +.. + +.. date: 2025-10-28-02-46-56 +.. gh-issue: 139946 +.. nonce: aN3_uY +.. section: Library + +Error and warning keywords in ``argparse.ArgumentParser`` messages are now +colorized when color output is enabled, fixing a visual inconsistency in +which they remained plain text while other output was colorized. + +.. + +.. date: 2025-10-27-18-29-42 +.. gh-issue: 140590 +.. nonce: LT9HHn +.. section: Library + +Fix arguments checking for the :meth:`!functools.partial.__setstate__` that +may lead to internal state corruption and crash. Patch by Sergey Miryanov. + +.. + +.. date: 2025-10-27-16-01-41 +.. gh-issue: 125434 +.. nonce: qy0uRA +.. section: Library + +Display thread name in :mod:`faulthandler` on Windows. Patch by Victor +Stinner. + +.. + +.. date: 2025-10-27-13-49-31 +.. gh-issue: 140634 +.. nonce: ULng9G +.. section: Library + +Fix a reference counting bug in :meth:`!os.sched_param.__reduce__`. + +.. + +.. date: 2025-10-27-00-40-49 +.. gh-issue: 140650 +.. nonce: DYJPJ9 +.. section: Library + +Fix an issue where closing :class:`io.BufferedWriter` could crash if the +closed attribute raised an exception on access or could not be converted to +a boolean. + +.. + +.. date: 2025-10-26-16-24-12 +.. gh-issue: 140633 +.. nonce: ioayC1 +.. section: Library + +Ignore :exc:`AttributeError` when setting a module's ``__file__`` attribute +when loading an extension module packaged as Apple Framework. + +.. + +.. date: 2025-10-25-22-55-07 +.. gh-issue: 140601 +.. nonce: In3MlS +.. section: Library + +:func:`xml.etree.ElementTree.iterparse` now emits a :exc:`ResourceWarning` +when the iterator is not explicitly closed and was opened with a filename. +This helps developers identify and fix resource leaks. Patch by Osama +Abdelkader. + +.. + +.. date: 2025-10-25-21-26-16 +.. gh-issue: 140593 +.. nonce: OxlLc9 +.. section: Library + +:mod:`xml.parsers.expat`: Fix a memory leak that could affect users with +:meth:`~xml.parsers.expat.xmlparser.ElementDeclHandler` set to a custom +element declaration handler. Patch by Sebastian Pipping. + +.. + +.. date: 2025-10-25-21-04-00 +.. gh-issue: 140607 +.. nonce: oOZGxS +.. section: Library + +Inside :meth:`io.RawIOBase.read`, validate that the count of bytes returned +by :meth:`io.RawIOBase.readinto` is valid (inside the provided buffer). + +.. + +.. date: 2025-10-23-19-39-16 +.. gh-issue: 138162 +.. nonce: Znw5DN +.. section: Library + +Fix :class:`logging.LoggerAdapter` with ``merge_extra=True`` and without the +*extra* argument. + +.. + +.. date: 2025-10-23-13-42-15 +.. gh-issue: 140481 +.. nonce: XKxWpq +.. section: Library + +Improve error message when trying to iterate a Tk widget, image or font. + +.. + +.. date: 2025-10-23-12-12-22 +.. gh-issue: 138774 +.. nonce: mnh2gU +.. section: Library + +:func:`ast.unparse` now generates full source code when handling +:class:`ast.Interpolation` nodes that do not have a specified source. + +.. + +.. date: 2025-10-22-20-52-13 +.. gh-issue: 140474 +.. nonce: xIWlip +.. section: Library + +Fix memory leak in :class:`array.array` when creating arrays from an empty +:class:`str` and the ``u`` type code. + +.. + +.. date: 2025-10-22-12-56-57 +.. gh-issue: 140448 +.. nonce: GsEkXD +.. section: Library + +Change the default of ``suggest_on_error`` to ``True`` in +``argparse.ArgumentParser``. + +.. + +.. date: 2025-10-21-15-54-13 +.. gh-issue: 137530 +.. nonce: ZyIVUH +.. section: Library + +:mod:`dataclasses` Fix annotations for generated ``__init__`` methods by +replacing the annotations that were in-line in the generated source code +with ``__annotate__`` functions attached to the methods. + +.. + +.. date: 2025-10-20-12-33-49 +.. gh-issue: 140348 +.. nonce: SAKnQZ +.. section: Library + +Fix regression in Python 3.14.0 where using the ``|`` operator on a +:class:`typing.Union` object combined with an object that is not a type +would raise an error. + +.. + +.. date: 2025-10-18-15-20-25 +.. gh-issue: 76007 +.. nonce: SNUzRq +.. section: Library + +:mod:`decimal`: Deprecate ``__version__`` and replace with +:data:`decimal.SPEC_VERSION`. + +.. + +.. date: 2025-10-18-14-30-21 +.. gh-issue: 76007 +.. nonce: peEgcr +.. section: Library + +Deprecate ``__version__`` from a :mod:`imaplib`. Patch by Hugo van Kemenade. + +.. + +.. date: 2025-10-17-23-58-11 +.. gh-issue: 140272 +.. nonce: lhY8uS +.. section: Library + +Fix memory leak in the :meth:`!clear` method of the :mod:`dbm.gnu` database. + +.. + +.. date: 2025-10-17-20-42-38 +.. gh-issue: 129117 +.. nonce: X9jr4p +.. section: Library + +:mod:`unicodedata`: Add :func:`~unicodedata.isxidstart` and +:func:`~unicodedata.isxidcontinue` functions to check whether a character +can start or continue a `Unicode Standard Annex #31 +`_ identifier. + +.. + +.. date: 2025-10-17-12-33-01 +.. gh-issue: 140251 +.. nonce: esM-OX +.. section: Library + +Colorize the default import statement ``import asyncio`` in asyncio REPL. + +.. + +.. date: 2025-10-16-22-49-16 +.. gh-issue: 140212 +.. nonce: llBNd0 +.. section: Library + +Calendar's HTML formatting now accepts year and month as options. +Previously, running ``python -m calendar -t html 2025 10`` would result in +an error message. It now generates an HTML document displaying the calendar +for the specified month. Contributed by Pål Grønås Drange. + +.. + +.. date: 2025-10-16-17-17-20 +.. gh-issue: 135801 +.. nonce: faH3fa +.. section: Library + +Improve filtering by module in :func:`warnings.warn_explicit` if no *module* +argument is passed. It now tests the module regular expression in the +warnings filter not only against the filename with ``.py`` stripped, but +also against module names constructed starting from different parent +directories of the filename (with ``/__init__.py``, ``.py`` and, on Windows, +``.pyw`` stripped). + +.. + +.. date: 2025-10-16-16-10-11 +.. gh-issue: 139707 +.. nonce: zR6Qtn +.. section: Library + +Improve :exc:`ModuleNotFoundError` error message when a :term:`standard +library` module is missing. + +.. + +.. date: 2025-10-15-21-42-13 +.. gh-issue: 140041 +.. nonce: _Fka2j +.. section: Library + +Fix import of :mod:`ctypes` on Android and Cygwin when ABI flags are +present. + +.. + +.. date: 2025-10-15-20-47-04 +.. gh-issue: 140120 +.. nonce: 3gffZq +.. section: Library + +Fixed a memory leak in :mod:`hmac` when it was using the hacl-star backend. +Discovered by ``@ashm-dev`` using AddressSanitizer. + +.. + +.. date: 2025-10-15-17-23-51 +.. gh-issue: 140141 +.. nonce: j2mUDB +.. section: Library + +The :py:class:`importlib.metadata.PackageNotFoundError` traceback raised +when ``importlib.metadata.Distribution.from_name`` cannot discover a +distribution no longer includes a transient :exc:`StopIteration` exception +trace. + +Contributed by Bartosz Sławecki in :gh:`140142`. + +.. + +.. date: 2025-10-15-15-10-34 +.. gh-issue: 140166 +.. nonce: NtxRez +.. section: Library + +:mod:`mimetypes`: Per the `IANA assignment +`_, update +the MIME type for the ``.texi`` and ``.texinfo`` file formats to +``application/texinfo``, instead of ``application/x-texinfo``. + +.. + +.. date: 2025-10-15-02-26-50 +.. gh-issue: 140135 +.. nonce: 54JYfM +.. section: Library + +Speed up :meth:`io.RawIOBase.readall` by using PyBytesWriter API (about 4x +faster) + +.. + +.. date: 2025-10-14-20-27-06 +.. gh-issue: 76007 +.. nonce: 2NcUbo +.. section: Library + +:mod:`zlib`: Deprecate ``__version__`` and schedule for removal in Python +3.20. + +.. + +.. date: 2025-10-13-11-25-41 +.. gh-issue: 136702 +.. nonce: uvLGK1 +.. section: Library + +:mod:`encodings`: Deprecate passing a non-ascii *encoding* name to +:func:`encodings.normalize_encoding` and schedule removal of support for +Python 3.17. + +.. + +.. date: 2025-10-11-09-07-06 +.. gh-issue: 139940 +.. nonce: g54efZ +.. section: Library + +Print clearer error message when using ``pdb`` to attach to a non-existing +process. + +.. + +.. date: 2025-10-02-22-29-00 +.. gh-issue: 139462 +.. nonce: VZXUHe +.. section: Library + +When a child process in a :class:`concurrent.futures.ProcessPoolExecutor` +terminates abruptly, the resulting traceback will now tell you the PID and +exit code of the terminated process. Contributed by Jonathan Berg. + +.. + +.. date: 2025-09-30-12-52-54 +.. gh-issue: 63161 +.. nonce: mECM1A +.. section: Library + +Fix :func:`tokenize.detect_encoding`. Support non-UTF-8 shebang and comments +if non-UTF-8 encoding is specified. Detect decoding error for non-UTF-8 +encoding. Detect null bytes in source code. + +.. + +.. date: 2025-09-25-20-16-10 +.. gh-issue: 101828 +.. nonce: yTxJlJ +.. section: Library + +Fix ``'shift_jisx0213'``, ``'shift_jis_2004'``, ``'euc_jisx0213'`` and +``'euc_jis_2004'`` codecs truncating null chars as they were treated as part +of multi-character sequences. + +.. + +.. date: 2025-09-23-09-46-46 +.. gh-issue: 139246 +.. nonce: pzfM-w +.. section: Library + +fix: paste zero-width in default repl width is wrong. + +.. + +.. date: 2025-09-18-21-25-41 +.. gh-issue: 83714 +.. nonce: TQjDWZ +.. section: Library + +Implement :func:`os.statx` on Linux kernel versions 4.11 and later with +glibc versions 2.28 and later. Contributed by Jeffrey Bosboom and Victor +Stinner. + +.. + +.. date: 2025-09-15-21-03-11 +.. gh-issue: 138891 +.. nonce: oZFdtR +.. section: Library + +Fix ``SyntaxError`` when ``inspect.get_annotations(f, eval_str=True)`` is +called on a function annotated with a :pep:`646` ``star_expression`` + +.. + +.. date: 2025-09-13-12-19-17 +.. gh-issue: 138859 +.. nonce: PxjIoN +.. section: Library + +Fix generic type parameterization raising a :exc:`TypeError` when omitting a +:class:`ParamSpec` that has a default which is not a list of types. + +.. + +.. date: 2025-09-12-09-34-37 +.. gh-issue: 138764 +.. nonce: mokHoY +.. section: Library + +Prevent :func:`annotationlib.call_annotate_function` from calling +``__annotate__`` functions that don't support ``VALUE_WITH_FAKE_GLOBALS`` in +a fake globals namespace with empty globals. + +Make ``FORWARDREF`` and ``STRING`` annotations fall back to using ``VALUE`` +annotations in the case that neither their own format, nor +``VALUE_WITH_FAKE_GLOBALS`` are supported. + +.. + +.. date: 2025-09-11-15-03-37 +.. gh-issue: 138775 +.. nonce: w7rnSx +.. section: Library + +Use of ``python -m`` with :mod:`base64` has been fixed to detect input from +a terminal so that it properly notices EOF. + +.. + +.. date: 2025-09-03-20-18-39 +.. gh-issue: 98896 +.. nonce: tjez89 +.. section: Library + +Fix a failure in multiprocessing resource_tracker when SharedMemory names +contain colons. Patch by Rani Pinchuk. + +.. + +.. date: 2025-09-03-18-26-07 +.. gh-issue: 138425 +.. nonce: cVE9Ho +.. section: Library + +Fix partial evaluation of :class:`annotationlib.ForwardRef` objects which +rely on names defined as globals. + +.. + +.. date: 2025-08-26-08-17-56 +.. gh-issue: 138151 +.. nonce: I6CdAk +.. section: Library + +In :mod:`annotationlib`, improve evaluation of forward references to +nonlocal variables that are not yet defined when the annotations are +initially evaluated. + +.. + +.. date: 2025-08-15-20-35-30 +.. gh-issue: 69528 +.. nonce: qc-Eh_ +.. section: Library + +The :attr:`~io.FileIO.mode` attribute of files opened in the ``'wb+'`` mode +is now ``'wb+'`` instead of ``'rb+'``. + +.. + +.. date: 2025-08-11-04-52-18 +.. gh-issue: 137627 +.. nonce: Ku5Yi2 +.. section: Library + +Speed up :meth:`csv.Sniffer.sniff` delimiter detection by up to 1.6x. + +.. + +.. date: 2025-07-14-09-33-17 +.. gh-issue: 55531 +.. nonce: Gt2e12 +.. section: Library + +:mod:`encodings`: Improve :func:`~encodings.normalize_encoding` performance +by implementing the function in C using the private +``_Py_normalize_encoding`` which has been modified to make lowercase +conversion optional. + +.. + +.. date: 2025-07-01-04-57-57 +.. gh-issue: 136057 +.. nonce: 4-t596 +.. section: Library + +Fixed the bug in :mod:`pdb` and :mod:`bdb` where ``next`` and ``step`` can't +go over the line if a loop exists in the line. + +.. + +.. date: 2025-06-29-22-01-00 +.. gh-issue: 133390 +.. nonce: I1DW_3 +.. section: Library + +Support table, index, trigger, view, column, function, and schema completion +for :mod:`sqlite3`'s :ref:`command-line interface `. + +.. + +.. date: 2025-06-10-18-02-29 +.. gh-issue: 135307 +.. nonce: fXGrcK +.. section: Library + +:mod:`email`: Fix exception in ``set_content()`` when encoding text and +max_line_length is set to ``0`` or ``None`` (unlimited). + +.. + +.. date: 2025-05-10-15-10-54 +.. gh-issue: 133789 +.. nonce: I-ZlUX +.. section: Library + +Fix unpickling of :mod:`pathlib` objects that were pickled in Python 3.13. + +.. + +.. date: 2025-05-07-22-09-28 +.. gh-issue: 133601 +.. nonce: 9kUL3P +.. section: Library + +Remove deprecated :func:`!typing.no_type_check_decorator`. + +.. + +.. date: 2025-04-18-18-08-05 +.. gh-issue: 132686 +.. nonce: 6kV_Gs +.. section: Library + +Add parameters *inherit_class_doc* and *fallback_to_class_doc* for +:func:`inspect.getdoc`. + +.. + +.. date: 2025-03-12-18-57-10 +.. gh-issue: 131116 +.. nonce: uTpwXZ +.. section: Library + +:func:`inspect.getdoc` now correctly returns an inherited docstring on +:class:`~functools.cached_property` objects if none is given in a subclass. + +.. + +.. date: 2025-03-04-17-19-26 +.. gh-issue: 130693 +.. nonce: Kv01r8 +.. section: Library + +Add support for ``-nolinestop``, and ``-strictlimits`` options to +:meth:`!tkinter.Text.search`. Also add the :meth:`!tkinter.Text.search_all` +method for ``-all`` and ``-overlap`` options. + +.. + +.. date: 2024-08-08-12-39-36 +.. gh-issue: 122255 +.. nonce: J_gU8Y +.. section: Library + +In the :mod:`linecache` module and in the Python implementation of the +:mod:`warnings` module, a ``DeprecationWarning`` is issued when +``mod.__loader__`` differs from ``mod.__spec__.loader`` (like in the C +implementation of the :mod:`!warnings` module). + +.. + +.. date: 2024-06-26-16-16-43 +.. gh-issue: 121011 +.. nonce: qW54eh +.. section: Library + +:func:`math.log` now supports arbitrary large integer-like arguments in the +same way as arbitrary large integer arguments. + +.. + +.. date: 2024-05-28-17-14-30 +.. gh-issue: 119668 +.. nonce: RrIGpn +.. section: Library + +Publicly expose and document :class:`importlib.machinery.NamespacePath`. + +.. + +.. date: 2023-03-21-10-59-40 +.. gh-issue: 102431 +.. nonce: eUDnf4 +.. section: Library + +Clarify constraints for "logical" arguments in methods of +:class:`decimal.Context`. + +.. + +.. date: 2019-06-02-13-56-16 +.. gh-issue: 81313 +.. nonce: axawSH +.. section: Library + +Add the :mod:`math.integer` module (:pep:`791`). + +.. + +.. date: 2025-11-15-01-21-00 +.. gh-issue: 141579 +.. nonce: aB7cD9 +.. section: Core and Builtins + +Fix :func:`sys.activate_stack_trampoline` to properly support the +``perf_jit`` backend. Patch by Pablo Galindo. + +.. + +.. date: 2025-11-14-16-25-15 +.. gh-issue: 114203 +.. nonce: n3tlQO +.. section: Core and Builtins + +Skip locking if object is already locked by two-mutex critical section. + +.. + +.. date: 2025-11-14-00-19-45 +.. gh-issue: 141528 +.. nonce: VWdax1 +.. section: Core and Builtins + +Suggest using :meth:`concurrent.interpreters.Interpreter.close` instead of +the private ``_interpreters.destroy`` function when warning about remaining +subinterpreters. Patch by Sergey Miryanov. + +.. + +.. date: 2025-11-11-13-40-45 +.. gh-issue: 141367 +.. nonce: I5KY7F +.. section: Core and Builtins + +Specialize ``CALL_LIST_APPEND`` instruction only for lists, not for list +subclasses, to avoid unnecessary deopt. Patch by Mikhail Efimov. + +.. + +.. date: 2025-11-10-23-07-06 +.. gh-issue: 141312 +.. nonce: H-58GB +.. section: Core and Builtins + +Fix the assertion failure in the ``__setstate__`` method of the range +iterator when a non-integer argument is passed. Patch by Sergey Miryanov. + +.. + +.. date: 2025-11-05-19-50-37 +.. gh-issue: 140643 +.. nonce: QCEOqG +.. section: Core and Builtins + +Add support for ```` and ```` frames to +:mod:`!profiling.sampling` output to denote active garbage collection and +calls to native code. + +.. + +.. date: 2025-11-04-12-18-06 +.. gh-issue: 140942 +.. nonce: GYns6n +.. section: Core and Builtins + +Add ``.cjs`` to :mod:`mimetypes` to give CommonJS modules a MIME type of +``application/node``. + +.. + +.. date: 2025-11-04-04-57-24 +.. gh-issue: 140479 +.. nonce: lwQ2v2 +.. section: Core and Builtins + +Update JIT compilation to use LLVM 21 at build time. + +.. + +.. date: 2025-11-03-17-21-38 +.. gh-issue: 140939 +.. nonce: FVboAw +.. section: Core and Builtins + +Fix memory leak when :class:`bytearray` or :class:`bytes` is formated with +the ``%*b`` format with a large width that results in a :exc:`MemoryError`. + +.. + +.. date: 2025-11-02-15-28-33 +.. gh-issue: 140260 +.. nonce: JNzlGz +.. section: Core and Builtins + +Fix :mod:`struct` data race in endian table initialization with +subinterpreters. Patch by Shamil Abdulaev. + +.. + +.. date: 2025-11-02-12-47-38 +.. gh-issue: 140530 +.. nonce: S934bp +.. section: Core and Builtins + +Fix a reference leak when ``raise exc from cause`` fails. Patch by Bénédikt +Tran. + +.. + +.. date: 2025-10-31-14-03-42 +.. gh-issue: 90344 +.. nonce: gvZigO +.. section: Core and Builtins + +Replace :class:`io.IncrementalNewlineDecoder` with non incremental newline +decoders in codebase where :meth:`!io.IncrementalNewlineDecoder.decode` was +being called once. + +.. + +.. date: 2025-10-29-20-59-10 +.. gh-issue: 140373 +.. nonce: -uoaPP +.. section: Core and Builtins + +Correctly emit ``PY_UNWIND`` event when generator object is closed. Patch by +Mikhail Efimov. + +.. + +.. date: 2025-10-29-11-31-59 +.. gh-issue: 140729 +.. nonce: t9JsNt +.. section: Core and Builtins + +Fix pickling error in the sampling profiler when using +``concurrent.futures.ProcessPoolExecutor`` script can not be properly +pickled and executed in worker processes. + +.. + +.. date: 2025-10-25-21-31-43 +.. gh-issue: 131527 +.. nonce: V-JVNP +.. section: Core and Builtins + +Dynamic borrow checking for stackrefs is added to ``Py_STACKREF_DEBUG`` +mode. Patch by Mikhail Efimov. + +.. + +.. date: 2025-10-25-17-36-46 +.. gh-issue: 140576 +.. nonce: kj0SCY +.. section: Core and Builtins + +Fixed crash in :func:`tokenize.generate_tokens` in case of specific +incorrect input. Patch by Mikhail Efimov. + +.. + +.. date: 2025-10-25-07-25-52 +.. gh-issue: 140544 +.. nonce: lwjtQe +.. section: Core and Builtins + +Speed up accessing interpreter state by caching it in a thread local +variable. Patch by Kumar Aditya. + +.. + +.. date: 2025-10-24-20-42-33 +.. gh-issue: 140551 +.. nonce: -9swrl +.. section: Core and Builtins + +Fixed crash in :class:`dict` if :meth:`dict.clear` is called at the lookup +stage. Patch by Mikhail Efimov and Inada Naoki. + +.. + +.. date: 2025-10-24-20-16-42 +.. gh-issue: 140517 +.. nonce: cqun-K +.. section: Core and Builtins + +Fixed a reference leak when iterating over the result of :func:`map` with +``strict=True`` when the input iterables have different lengths. Patch by +Mikhail Efimov. + +.. + +.. date: 2025-10-24-14-29-12 +.. gh-issue: 133467 +.. nonce: A5d6TM +.. section: Core and Builtins + +Fix race when updating :attr:`!type.__bases__` that could allow a read of +:attr:`!type.__base__` to observe an inconsistent value on the free threaded +build. + +.. + +.. date: 2025-10-23-16-05-50 +.. gh-issue: 140471 +.. nonce: Ax_aXn +.. section: Core and Builtins + +Fix potential buffer overflow in :class:`ast.AST` node initialization when +encountering malformed :attr:`~ast.AST._fields` containing non-:class:`str`. + +.. + +.. date: 2025-10-22-23-26-37 +.. gh-issue: 140443 +.. nonce: wT5i1A +.. section: Core and Builtins + +The logarithm functions (such as :func:`math.log10` and :func:`math.log`) +may now produce slightly different results for extremely large integers that +cannot be converted to floats without overflow. These results are generally +more accurate, with reduced worst-case error and a tighter overall error +distribution. + +.. + +.. date: 2025-10-22-17-22-22 +.. gh-issue: 140431 +.. nonce: m8D_A- +.. section: Core and Builtins + +Fix a crash in Python's :term:`garbage collector ` due +to partially initialized :term:`coroutine` objects when coroutine origin +tracking depth is enabled (:func:`sys.set_coroutine_origin_tracking_depth`). + +.. + +.. date: 2025-10-22-12-48-05 +.. gh-issue: 140476 +.. nonce: F3-d1P +.. section: Core and Builtins + +Optimize :c:func:`PySet_Add` for :class:`frozenset` in :term:`free threaded +` build. + +.. + +.. date: 2025-10-22-11-30-16 +.. gh-issue: 135904 +.. nonce: 3WE5oW +.. section: Core and Builtins + +Add special labels to the assembly created during stencil creation to +support relocations that the native object file format does not support. +Specifically, 19 bit branches for AArch64 in Mach-O object files. + +.. + +.. date: 2025-10-21-09-20-03 +.. gh-issue: 140398 +.. nonce: SoABwJ +.. section: Core and Builtins + +Fix memory leaks in :mod:`readline` functions +:func:`~readline.read_init_file`, :func:`~readline.read_history_file`, +:func:`~readline.write_history_file`, and +:func:`~readline.append_history_file` when :c:func:`PySys_Audit` fails. + +.. + +.. date: 2025-10-21-06-51-50 +.. gh-issue: 140406 +.. nonce: 0gJs8M +.. section: Core and Builtins + +Fix memory leak when an object's :meth:`~object.__hash__` method returns an +object that isn't an :class:`int`. + +.. + +.. date: 2025-10-20-11-24-36 +.. gh-issue: 140358 +.. nonce: UQuKdV +.. section: Core and Builtins + +Restore elapsed time and unreachable object count in GC debug output. These +were inadvertently removed during a refactor of ``gc.c``. The debug log now +again reports elapsed collection time and the number of unreachable objects. +Contributed by Pål Grønås Drange. + +.. + +.. date: 2025-10-19-10-32-28 +.. gh-issue: 136895 +.. nonce: HfsEh0 +.. section: Core and Builtins + +Update JIT compilation to use LLVM 20 at build time. + +.. + +.. date: 2025-10-18-21-50-44 +.. gh-issue: 139109 +.. nonce: 9QQOzN +.. section: Core and Builtins + +A new tracing frontend for the JIT compiler has been implemented. Patch by +Ken Jin. Design for CPython by Ken Jin, Mark Shannon and Brandt Bucher. + +.. + +.. date: 2025-10-18-21-29-45 +.. gh-issue: 140306 +.. nonce: xS5CcS +.. section: Core and Builtins + +Fix memory leaks in cross-interpreter channel operations and shared +namespace handling. + +.. + +.. date: 2025-10-18-19-52-20 +.. gh-issue: 116738 +.. nonce: NLJW0L +.. section: Core and Builtins + +Make _suggestions module thread-safe on the :term:`free threaded ` build. + +.. + +.. date: 2025-10-18-18-08-36 +.. gh-issue: 140301 +.. nonce: m-2HxC +.. section: Core and Builtins + +Fix memory leak of ``PyConfig`` in subinterpreters. + +.. + +.. date: 2025-10-17-20-23-19 +.. gh-issue: 140257 +.. nonce: 8Txmem +.. section: Core and Builtins + +Fix data race between interpreter_clear() and take_gil() on eval_breaker +during finalization with daemon threads. + +.. + +.. date: 2025-10-17-18-03-12 +.. gh-issue: 139951 +.. nonce: IdwM2O +.. section: Core and Builtins + +Fixes a regression in GC performance for a growing heap composed mostly of +small tuples. + +* Counts number of actually tracked objects, instead of trackable objects. + This ensures that untracking tuples has the desired effect of reducing GC overhead. +* Does not track most untrackable tuples during creation. + This prevents large numbers of small tuples causing excessive GCs. + +.. + +.. date: 2025-10-17-14-38-10 +.. gh-issue: 140253 +.. nonce: gCqFaL +.. section: Core and Builtins + +Wrong placement of a double-star pattern inside a mapping pattern now throws +a specialized syntax error. Contributed by Bartosz Sławecki in :gh:`140253`. + +.. + +.. date: 2025-10-16-21-47-00 +.. gh-issue: 140104 +.. nonce: A8SQIm +.. section: Core and Builtins + +Fix a bug with exception handling in the JIT. Patch by Ken Jin. Bug reported +by Daniel Diniz. + +.. + +.. date: 2025-10-15-17-12-32 +.. gh-issue: 140149 +.. nonce: cy1m3d +.. section: Core and Builtins + +Speed up parsing bytes literals concatenation by using PyBytesWriter API and +a single memory allocation (about 3x faster). + +.. + +.. date: 2025-10-15-00-21-40 +.. gh-issue: 140061 +.. nonce: J0XeDV +.. section: Core and Builtins + +Fixing the checking of whether an object is uniquely referenced to ensure +free-threaded compatibility. Patch by Sergey Miryanov. + +.. + +.. date: 2025-10-14-20-18-31 +.. gh-issue: 140080 +.. nonce: 8ROjxW +.. section: Core and Builtins + +Fix hang during finalization when attempting to call :mod:`atexit` handlers +under no memory. + +.. + +.. date: 2025-10-14-18-24-16 +.. gh-issue: 139871 +.. nonce: SWtuUz +.. section: Core and Builtins + +Update :class:`bytearray` to use a :class:`bytes` under the hood as its +buffer and add :func:`bytearray.take_bytes` to take it out. + +.. + +.. date: 2025-10-14-17-07-37 +.. gh-issue: 140067 +.. nonce: ID2gOm +.. section: Core and Builtins + +Fix memory leak in sub-interpreter creation. + +.. + +.. date: 2025-10-13-13-54-19 +.. gh-issue: 139914 +.. nonce: M-y_3E +.. section: Core and Builtins + +Restore support for HP PA-RISC, which has an upwards-growing stack. + +.. + +.. date: 2025-10-12-01-12-12 +.. gh-issue: 139817 +.. nonce: PAn-8Z +.. section: Core and Builtins + +Attribute ``__qualname__`` is added to :class:`typing.TypeAliasType`. Patch +by Mikhail Efimov. + +.. + +.. date: 2025-10-06-14-19-47 +.. gh-issue: 135801 +.. nonce: OhxEZS +.. section: Core and Builtins + +Many functions related to compiling or parsing Python code, such as +:func:`compile`, :func:`ast.parse`, :func:`symtable.symtable`, and +:func:`importlib.abc.InspectLoader.source_to_code` now allow to specify the +module name. It is needed to unambiguous :ref:`filter ` +syntax warnings by module name. + +.. + +.. date: 2025-10-06-10-03-37 +.. gh-issue: 139640 +.. nonce: gY5oTb2 +.. section: Core and Builtins + +:func:`ast.parse` no longer emits syntax warnings for +``return``/``break``/``continue`` in ``finally`` (see :pep:`765`) -- they +are only emitted during compilation. + +.. + +.. date: 2025-10-06-10-03-37 +.. gh-issue: 139640 +.. nonce: gY5oTb +.. section: Core and Builtins + +Fix swallowing some syntax warnings in different modules if they +accidentally have the same message and are emitted from the same line. Fix +duplicated warnings in the ``finally`` block. + +.. + +.. date: 2025-10-03-17-51-43 +.. gh-issue: 139475 +.. nonce: _684ED +.. section: Core and Builtins + +Changes in stackref debugging mode when ``Py_STACKREF_DEBUG`` is set. We use +the same pattern of refcounting for stackrefs as in production build. + +.. + +.. date: 2025-09-23-21-01-12 +.. gh-issue: 139269 +.. nonce: 1rIaxy +.. section: Core and Builtins + +Fix undefined behavior when using unaligned store in JIT's ``patch_*`` +functions. + +.. + +.. date: 2025-09-15-13-06-11 +.. gh-issue: 138944 +.. nonce: PeCgLb +.. section: Core and Builtins + +Fix :exc:`SyntaxError` message when invalid syntax appears on the same line +as a valid ``import ... as ...`` or ``from ... import ... as ...`` +statement. Patch by Brian Schubert. + +.. + +.. date: 2025-09-13-01-23-25 +.. gh-issue: 138857 +.. nonce: YQ5gdc +.. section: Core and Builtins + +Improve :exc:`SyntaxError` message for ``case`` keyword placed outside +:keyword:`match` body. + +.. + +.. date: 2025-07-29-17-51-14 +.. gh-issue: 131253 +.. nonce: GpRjWy +.. section: Core and Builtins + +Support the ``--enable-pystats`` build option for the free-threaded build. + +.. + +.. date: 2025-07-08-00-41-46 +.. gh-issue: 136327 +.. nonce: 7AiTb_ +.. section: Core and Builtins + +Errors when calling functions with invalid values after ``*`` and ``**`` now +do not include the function name. Patch by Ilia Solin. + +.. + +.. date: 2025-06-24-13-12-58 +.. gh-issue: 134786 +.. nonce: MF0VVk +.. section: Core and Builtins + +If :c:macro:`Py_TPFLAGS_MANAGED_DICT` and +:c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` are used, then +:c:macro:`Py_TPFLAGS_HAVE_GC` must be used as well. + +.. + +.. date: 2025-11-10-11-26-26 +.. gh-issue: 141341 +.. nonce: OsO6-y +.. section: C API + +On Windows, rename the ``COMPILER`` macro to ``_Py_COMPILER`` to avoid name +conflicts. Patch by Victor Stinner. + +.. + +.. date: 2025-11-08-10-51-50 +.. gh-issue: 116146 +.. nonce: pCmx6L +.. section: C API + +Add a new :c:func:`PyImport_CreateModuleFromInitfunc` C-API for creating a +module from a *spec* and *initfunc*. Patch by Itamar Oren. + +.. + +.. date: 2025-11-06-06-28-14 +.. gh-issue: 141042 +.. nonce: brOioJ +.. section: C API + +Make qNaN in :c:func:`PyFloat_Pack2` and :c:func:`PyFloat_Pack4`, if while +conversion to a narrower precision floating-point format --- the remaining +after truncation payload will be zero. Patch by Sergey B Kirpichev. + +.. + +.. date: 2025-11-05-05-45-49 +.. gh-issue: 141004 +.. nonce: N9Ooh9 +.. section: C API + +:c:macro:`!Py_MATH_El` and :c:macro:`!Py_MATH_PIl` are deprecated. + +.. + +.. date: 2025-11-05-04-38-16 +.. gh-issue: 141004 +.. nonce: rJL43P +.. section: C API + +The :c:macro:`!Py_INFINITY` macro is :term:`soft deprecated`. + +.. + +.. date: 2025-10-26-16-45-28 +.. gh-issue: 140556 +.. nonce: s__Dae +.. section: C API + +:pep:`793`: Add a new entry point for C extension modules, +``PyModExport_``. + +.. + +.. date: 2025-10-26-16-45-06 +.. gh-issue: 140487 +.. nonce: fGOqss +.. section: C API + +Fix :c:macro:`Py_RETURN_NOTIMPLEMENTED` in limited C API 3.11 and older: +don't treat ``Py_NotImplemented`` as immortal. Patch by Victor Stinner. + +.. + +.. date: 2025-10-15-15-59-59 +.. gh-issue: 140153 +.. nonce: BO7sH4 +.. section: C API + +Fix :c:func:`Py_REFCNT` definition on limited C API 3.11-3.13. Patch by +Victor Stinner. + +.. + +.. date: 2025-10-06-22-17-47 +.. gh-issue: 139653 +.. nonce: 6-1MOd +.. section: C API + +Add :c:func:`PyUnstable_ThreadState_SetStackProtection` and +:c:func:`PyUnstable_ThreadState_ResetStackProtection` functions to set the +stack protection base address and stack protection size of a Python thread +state. Patch by Victor Stinner. + +.. + +.. date: 2025-10-31-13-20-16 +.. gh-issue: 140454 +.. nonce: gF6dCe +.. section: Build + +When building the JIT, match the jit_stencils filename expectations in +Makefile with the generator script. This avoid needless JIT recompilation +during ``make install``. + +.. + +.. date: 2025-10-29-12-30-38 +.. gh-issue: 140768 +.. nonce: ITYrzw +.. section: Build + +Warn when the WASI SDK version doesn't match what's supported. + +.. + +.. date: 2025-10-25-08-07-06 +.. gh-issue: 140513 +.. nonce: 6OhLTs +.. section: Build + +Generate a clear compilation error when ``_Py_TAIL_CALL_INTERP`` is enabled +but either ``preserve_none`` or ``musttail`` is not supported. + +.. + +.. date: 2025-10-22-12-44-07 +.. gh-issue: 140475 +.. nonce: OhzQbR +.. section: Build + +Support WASI SDK 25. + +.. + +.. date: 2025-10-17-11-33-45 +.. gh-issue: 140239 +.. nonce: _k-GgW +.. section: Build + +Check ``statx`` availability only on Linux (including Android). + +.. + +.. date: 2025-10-16-11-30-53 +.. gh-issue: 140189 +.. nonce: YCrUyt +.. section: Build + +iOS builds were added to CI. + +.. + +.. date: 2025-08-10-22-28-06 +.. gh-issue: 137618 +.. nonce: FdNvIE +.. section: Build + +``PYTHON_FOR_REGEN`` now requires Python 3.10 to Python 3.15. Patch by Adam +Turner. diff --git a/Misc/NEWS.d/next/Build/2025-08-10-22-28-06.gh-issue-137618.FdNvIE.rst b/Misc/NEWS.d/next/Build/2025-08-10-22-28-06.gh-issue-137618.FdNvIE.rst deleted file mode 100644 index 0b56c4c8f68566..00000000000000 --- a/Misc/NEWS.d/next/Build/2025-08-10-22-28-06.gh-issue-137618.FdNvIE.rst +++ /dev/null @@ -1,2 +0,0 @@ -``PYTHON_FOR_REGEN`` now requires Python 3.10 to Python 3.15. -Patch by Adam Turner. diff --git a/Misc/NEWS.d/next/Build/2025-10-16-11-30-53.gh-issue-140189.YCrUyt.rst b/Misc/NEWS.d/next/Build/2025-10-16-11-30-53.gh-issue-140189.YCrUyt.rst deleted file mode 100644 index a1b81659242670..00000000000000 --- a/Misc/NEWS.d/next/Build/2025-10-16-11-30-53.gh-issue-140189.YCrUyt.rst +++ /dev/null @@ -1 +0,0 @@ -iOS builds were added to CI. diff --git a/Misc/NEWS.d/next/Build/2025-10-17-11-33-45.gh-issue-140239._k-GgW.rst b/Misc/NEWS.d/next/Build/2025-10-17-11-33-45.gh-issue-140239._k-GgW.rst deleted file mode 100644 index 713f022c994958..00000000000000 --- a/Misc/NEWS.d/next/Build/2025-10-17-11-33-45.gh-issue-140239._k-GgW.rst +++ /dev/null @@ -1 +0,0 @@ -Check ``statx`` availability only on Linux (including Android). diff --git a/Misc/NEWS.d/next/Build/2025-10-22-12-44-07.gh-issue-140475.OhzQbR.rst b/Misc/NEWS.d/next/Build/2025-10-22-12-44-07.gh-issue-140475.OhzQbR.rst deleted file mode 100644 index b4139024761815..00000000000000 --- a/Misc/NEWS.d/next/Build/2025-10-22-12-44-07.gh-issue-140475.OhzQbR.rst +++ /dev/null @@ -1 +0,0 @@ -Support WASI SDK 25. diff --git a/Misc/NEWS.d/next/Build/2025-10-25-08-07-06.gh-issue-140513.6OhLTs.rst b/Misc/NEWS.d/next/Build/2025-10-25-08-07-06.gh-issue-140513.6OhLTs.rst deleted file mode 100644 index 1035ebf8d781cf..00000000000000 --- a/Misc/NEWS.d/next/Build/2025-10-25-08-07-06.gh-issue-140513.6OhLTs.rst +++ /dev/null @@ -1,2 +0,0 @@ -Generate a clear compilation error when ``_Py_TAIL_CALL_INTERP`` is enabled but -either ``preserve_none`` or ``musttail`` is not supported. diff --git a/Misc/NEWS.d/next/Build/2025-10-29-12-30-38.gh-issue-140768.ITYrzw.rst b/Misc/NEWS.d/next/Build/2025-10-29-12-30-38.gh-issue-140768.ITYrzw.rst deleted file mode 100644 index 0009f83cd20d8f..00000000000000 --- a/Misc/NEWS.d/next/Build/2025-10-29-12-30-38.gh-issue-140768.ITYrzw.rst +++ /dev/null @@ -1 +0,0 @@ -Warn when the WASI SDK version doesn't match what's supported. diff --git a/Misc/NEWS.d/next/Build/2025-10-31-13-20-16.gh-issue-140454.gF6dCe.rst b/Misc/NEWS.d/next/Build/2025-10-31-13-20-16.gh-issue-140454.gF6dCe.rst deleted file mode 100644 index 4bb132ce01e170..00000000000000 --- a/Misc/NEWS.d/next/Build/2025-10-31-13-20-16.gh-issue-140454.gF6dCe.rst +++ /dev/null @@ -1,3 +0,0 @@ -When building the JIT, match the jit_stencils filename expectations in -Makefile with the generator script. This avoid needless JIT recompilation -during ``make install``. diff --git a/Misc/NEWS.d/next/C_API/2025-10-06-22-17-47.gh-issue-139653.6-1MOd.rst b/Misc/NEWS.d/next/C_API/2025-10-06-22-17-47.gh-issue-139653.6-1MOd.rst deleted file mode 100644 index cd3d5262fa0f3a..00000000000000 --- a/Misc/NEWS.d/next/C_API/2025-10-06-22-17-47.gh-issue-139653.6-1MOd.rst +++ /dev/null @@ -1,4 +0,0 @@ -Add :c:func:`PyUnstable_ThreadState_SetStackProtection` and -:c:func:`PyUnstable_ThreadState_ResetStackProtection` functions to set the -stack protection base address and stack protection size of a Python thread -state. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C_API/2025-10-15-15-59-59.gh-issue-140153.BO7sH4.rst b/Misc/NEWS.d/next/C_API/2025-10-15-15-59-59.gh-issue-140153.BO7sH4.rst deleted file mode 100644 index 502c48b6842d1c..00000000000000 --- a/Misc/NEWS.d/next/C_API/2025-10-15-15-59-59.gh-issue-140153.BO7sH4.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :c:func:`Py_REFCNT` definition on limited C API 3.11-3.13. Patch by -Victor Stinner. diff --git a/Misc/NEWS.d/next/C_API/2025-10-26-16-45-06.gh-issue-140487.fGOqss.rst b/Misc/NEWS.d/next/C_API/2025-10-26-16-45-06.gh-issue-140487.fGOqss.rst deleted file mode 100644 index 16b0d9d4084ba0..00000000000000 --- a/Misc/NEWS.d/next/C_API/2025-10-26-16-45-06.gh-issue-140487.fGOqss.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :c:macro:`Py_RETURN_NOTIMPLEMENTED` in limited C API 3.11 and older: -don't treat ``Py_NotImplemented`` as immortal. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C_API/2025-10-26-16-45-28.gh-issue-140556.s__Dae.rst b/Misc/NEWS.d/next/C_API/2025-10-26-16-45-28.gh-issue-140556.s__Dae.rst deleted file mode 100644 index 61da60903ee821..00000000000000 --- a/Misc/NEWS.d/next/C_API/2025-10-26-16-45-28.gh-issue-140556.s__Dae.rst +++ /dev/null @@ -1,2 +0,0 @@ -:pep:`793`: Add a new entry point for C extension modules, -``PyModExport_``. diff --git a/Misc/NEWS.d/next/C_API/2025-11-05-04-38-16.gh-issue-141004.rJL43P.rst b/Misc/NEWS.d/next/C_API/2025-11-05-04-38-16.gh-issue-141004.rJL43P.rst deleted file mode 100644 index a054f8eda6fb0b..00000000000000 --- a/Misc/NEWS.d/next/C_API/2025-11-05-04-38-16.gh-issue-141004.rJL43P.rst +++ /dev/null @@ -1 +0,0 @@ -The :c:macro:`!Py_INFINITY` macro is :term:`soft deprecated`. diff --git a/Misc/NEWS.d/next/C_API/2025-11-05-05-45-49.gh-issue-141004.N9Ooh9.rst b/Misc/NEWS.d/next/C_API/2025-11-05-05-45-49.gh-issue-141004.N9Ooh9.rst deleted file mode 100644 index 5f3ccd62016e11..00000000000000 --- a/Misc/NEWS.d/next/C_API/2025-11-05-05-45-49.gh-issue-141004.N9Ooh9.rst +++ /dev/null @@ -1 +0,0 @@ -:c:macro:`!Py_MATH_El` and :c:macro:`!Py_MATH_PIl` are deprecated. diff --git a/Misc/NEWS.d/next/C_API/2025-11-06-06-28-14.gh-issue-141042.brOioJ.rst b/Misc/NEWS.d/next/C_API/2025-11-06-06-28-14.gh-issue-141042.brOioJ.rst deleted file mode 100644 index 22a1aa1f405318..00000000000000 --- a/Misc/NEWS.d/next/C_API/2025-11-06-06-28-14.gh-issue-141042.brOioJ.rst +++ /dev/null @@ -1,3 +0,0 @@ -Make qNaN in :c:func:`PyFloat_Pack2` and :c:func:`PyFloat_Pack4`, if while -conversion to a narrower precision floating-point format --- the remaining -after truncation payload will be zero. Patch by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst b/Misc/NEWS.d/next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst deleted file mode 100644 index be8043e26ddda8..00000000000000 --- a/Misc/NEWS.d/next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add a new :c:func:`PyImport_CreateModuleFromInitfunc` C-API for creating a -module from a *spec* and *initfunc*. Patch by Itamar Oren. diff --git a/Misc/NEWS.d/next/C_API/2025-11-10-11-26-26.gh-issue-141341.OsO6-y.rst b/Misc/NEWS.d/next/C_API/2025-11-10-11-26-26.gh-issue-141341.OsO6-y.rst deleted file mode 100644 index 460923b4d62e22..00000000000000 --- a/Misc/NEWS.d/next/C_API/2025-11-10-11-26-26.gh-issue-141341.OsO6-y.rst +++ /dev/null @@ -1,2 +0,0 @@ -On Windows, rename the ``COMPILER`` macro to ``_Py_COMPILER`` to avoid name -conflicts. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-24-13-12-58.gh-issue-134786.MF0VVk.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-24-13-12-58.gh-issue-134786.MF0VVk.rst deleted file mode 100644 index 664e4d2db384ad..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-24-13-12-58.gh-issue-134786.MF0VVk.rst +++ /dev/null @@ -1,2 +0,0 @@ -If :c:macro:`Py_TPFLAGS_MANAGED_DICT` and :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` -are used, then :c:macro:`Py_TPFLAGS_HAVE_GC` must be used as well. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-00-41-46.gh-issue-136327.7AiTb_.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-00-41-46.gh-issue-136327.7AiTb_.rst deleted file mode 100644 index 3798e956c95ab6..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-00-41-46.gh-issue-136327.7AiTb_.rst +++ /dev/null @@ -1,2 +0,0 @@ -Errors when calling functions with invalid values after ``*`` and ``**`` now do not -include the function name. Patch by Ilia Solin. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-29-17-51-14.gh-issue-131253.GpRjWy.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-29-17-51-14.gh-issue-131253.GpRjWy.rst deleted file mode 100644 index 2826fad233058a..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-29-17-51-14.gh-issue-131253.GpRjWy.rst +++ /dev/null @@ -1 +0,0 @@ -Support the ``--enable-pystats`` build option for the free-threaded build. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-09-13-01-23-25.gh-issue-138857.YQ5gdc.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-09-13-01-23-25.gh-issue-138857.YQ5gdc.rst deleted file mode 100644 index 93510a9ceaf3c8..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-09-13-01-23-25.gh-issue-138857.YQ5gdc.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve :exc:`SyntaxError` message for ``case`` keyword placed outside -:keyword:`match` body. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-09-15-13-06-11.gh-issue-138944.PeCgLb.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-09-15-13-06-11.gh-issue-138944.PeCgLb.rst deleted file mode 100644 index 248585e2eba995..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-09-15-13-06-11.gh-issue-138944.PeCgLb.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix :exc:`SyntaxError` message when invalid syntax appears on the same line -as a valid ``import ... as ...`` or ``from ... import ... as ...`` -statement. Patch by Brian Schubert. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-09-23-21-01-12.gh-issue-139269.1rIaxy.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-09-23-21-01-12.gh-issue-139269.1rIaxy.rst deleted file mode 100644 index e36be529d2a5b9..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-09-23-21-01-12.gh-issue-139269.1rIaxy.rst +++ /dev/null @@ -1 +0,0 @@ -Fix undefined behavior when using unaligned store in JIT's ``patch_*`` functions. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-03-17-51-43.gh-issue-139475._684ED.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-03-17-51-43.gh-issue-139475._684ED.rst deleted file mode 100644 index f4d50b7d0207a0..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-03-17-51-43.gh-issue-139475._684ED.rst +++ /dev/null @@ -1,2 +0,0 @@ -Changes in stackref debugging mode when ``Py_STACKREF_DEBUG`` is set. We use -the same pattern of refcounting for stackrefs as in production build. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-10-03-37.gh-issue-139640.gY5oTb.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-10-03-37.gh-issue-139640.gY5oTb.rst deleted file mode 100644 index 396e40f0e1360b..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-10-03-37.gh-issue-139640.gY5oTb.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix swallowing some syntax warnings in different modules if they -accidentally have the same message and are emitted from the same line. -Fix duplicated warnings in the ``finally`` block. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-10-03-37.gh-issue-139640.gY5oTb2.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-10-03-37.gh-issue-139640.gY5oTb2.rst deleted file mode 100644 index b147b430ccccf5..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-10-03-37.gh-issue-139640.gY5oTb2.rst +++ /dev/null @@ -1,3 +0,0 @@ -:func:`ast.parse` no longer emits syntax warnings for -``return``/``break``/``continue`` in ``finally`` (see :pep:`765`) -- they are -only emitted during compilation. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-14-19-47.gh-issue-135801.OhxEZS.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-14-19-47.gh-issue-135801.OhxEZS.rst deleted file mode 100644 index 96226a7c525e80..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-06-14-19-47.gh-issue-135801.OhxEZS.rst +++ /dev/null @@ -1,6 +0,0 @@ -Many functions related to compiling or parsing Python code, such as -:func:`compile`, :func:`ast.parse`, :func:`symtable.symtable`, and -:func:`importlib.abc.InspectLoader.source_to_code` now allow to specify -the module name. -It is needed to unambiguous :ref:`filter ` syntax warnings -by module name. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-12-01-12-12.gh-issue-139817.PAn-8Z.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-12-01-12-12.gh-issue-139817.PAn-8Z.rst deleted file mode 100644 index b205d21edfec0c..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-12-01-12-12.gh-issue-139817.PAn-8Z.rst +++ /dev/null @@ -1,2 +0,0 @@ -Attribute ``__qualname__`` is added to :class:`typing.TypeAliasType`. -Patch by Mikhail Efimov. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-13-13-54-19.gh-issue-139914.M-y_3E.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-13-13-54-19.gh-issue-139914.M-y_3E.rst deleted file mode 100644 index 7529108d5d4772..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-13-13-54-19.gh-issue-139914.M-y_3E.rst +++ /dev/null @@ -1 +0,0 @@ -Restore support for HP PA-RISC, which has an upwards-growing stack. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-17-07-37.gh-issue-140067.ID2gOm.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-17-07-37.gh-issue-140067.ID2gOm.rst deleted file mode 100644 index 3c5a828101d9a8..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-17-07-37.gh-issue-140067.ID2gOm.rst +++ /dev/null @@ -1 +0,0 @@ -Fix memory leak in sub-interpreter creation. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-18-24-16.gh-issue-139871.SWtuUz.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-18-24-16.gh-issue-139871.SWtuUz.rst deleted file mode 100644 index d4b8578afe3afc..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-18-24-16.gh-issue-139871.SWtuUz.rst +++ /dev/null @@ -1,2 +0,0 @@ -Update :class:`bytearray` to use a :class:`bytes` under the hood as its buffer -and add :func:`bytearray.take_bytes` to take it out. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-20-18-31.gh-issue-140080.8ROjxW.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-20-18-31.gh-issue-140080.8ROjxW.rst deleted file mode 100644 index 0ddcea57f9d5b6..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-14-20-18-31.gh-issue-140080.8ROjxW.rst +++ /dev/null @@ -1 +0,0 @@ -Fix hang during finalization when attempting to call :mod:`atexit` handlers under no memory. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-15-00-21-40.gh-issue-140061.J0XeDV.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-15-00-21-40.gh-issue-140061.J0XeDV.rst deleted file mode 100644 index 7c3924195eb85f..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-15-00-21-40.gh-issue-140061.J0XeDV.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixing the checking of whether an object is uniquely referenced to ensure -free-threaded compatibility. Patch by Sergey Miryanov. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-15-17-12-32.gh-issue-140149.cy1m3d.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-15-17-12-32.gh-issue-140149.cy1m3d.rst deleted file mode 100644 index e98e28802cfee9..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-15-17-12-32.gh-issue-140149.cy1m3d.rst +++ /dev/null @@ -1,2 +0,0 @@ -Speed up parsing bytes literals concatenation by using PyBytesWriter API and -a single memory allocation (about 3x faster). diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-16-21-47-00.gh-issue-140104.A8SQIm.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-16-21-47-00.gh-issue-140104.A8SQIm.rst deleted file mode 100644 index 1c18cbc9ad0588..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-16-21-47-00.gh-issue-140104.A8SQIm.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a bug with exception handling in the JIT. Patch by Ken Jin. Bug reported -by Daniel Diniz. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-14-38-10.gh-issue-140253.gCqFaL.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-14-38-10.gh-issue-140253.gCqFaL.rst deleted file mode 100644 index 955dcac2e01564..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-14-38-10.gh-issue-140253.gCqFaL.rst +++ /dev/null @@ -1,2 +0,0 @@ -Wrong placement of a double-star pattern inside a mapping pattern now throws a specialized syntax error. -Contributed by Bartosz Sławecki in :gh:`140253`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-18-03-12.gh-issue-139951.IdwM2O.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-18-03-12.gh-issue-139951.IdwM2O.rst deleted file mode 100644 index e03996188a7e22..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-18-03-12.gh-issue-139951.IdwM2O.rst +++ /dev/null @@ -1,7 +0,0 @@ -Fixes a regression in GC performance for a growing heap composed mostly of -small tuples. - -* Counts number of actually tracked objects, instead of trackable objects. - This ensures that untracking tuples has the desired effect of reducing GC overhead. -* Does not track most untrackable tuples during creation. - This prevents large numbers of small tuples causing excessive GCs. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-20-23-19.gh-issue-140257.8Txmem.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-20-23-19.gh-issue-140257.8Txmem.rst deleted file mode 100644 index 50f7e0e48ae369..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-20-23-19.gh-issue-140257.8Txmem.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix data race between interpreter_clear() and take_gil() on eval_breaker -during finalization with daemon threads. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-18-08-36.gh-issue-140301.m-2HxC.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-18-08-36.gh-issue-140301.m-2HxC.rst deleted file mode 100644 index 8b1c81c04ece92..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-18-08-36.gh-issue-140301.m-2HxC.rst +++ /dev/null @@ -1 +0,0 @@ -Fix memory leak of ``PyConfig`` in subinterpreters. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-19-52-20.gh-issue-116738.NLJW0L.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-19-52-20.gh-issue-116738.NLJW0L.rst deleted file mode 100644 index bf323b870bc631..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-19-52-20.gh-issue-116738.NLJW0L.rst +++ /dev/null @@ -1,2 +0,0 @@ -Make _suggestions module thread-safe on the :term:`free threaded ` build. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-29-45.gh-issue-140306.xS5CcS.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-29-45.gh-issue-140306.xS5CcS.rst deleted file mode 100644 index 2178c4960636cb..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-29-45.gh-issue-140306.xS5CcS.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix memory leaks in cross-interpreter channel operations and shared -namespace handling. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst deleted file mode 100644 index 40b9d19ee42968..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst +++ /dev/null @@ -1 +0,0 @@ -A new tracing frontend for the JIT compiler has been implemented. Patch by Ken Jin. Design for CPython by Ken Jin, Mark Shannon and Brandt Bucher. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-19-10-32-28.gh-issue-136895.HfsEh0.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-19-10-32-28.gh-issue-136895.HfsEh0.rst deleted file mode 100644 index fffc264a8650e0..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-19-10-32-28.gh-issue-136895.HfsEh0.rst +++ /dev/null @@ -1 +0,0 @@ -Update JIT compilation to use LLVM 20 at build time. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-20-11-24-36.gh-issue-140358.UQuKdV.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-20-11-24-36.gh-issue-140358.UQuKdV.rst deleted file mode 100644 index 739228f7e36f20..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-20-11-24-36.gh-issue-140358.UQuKdV.rst +++ /dev/null @@ -1,4 +0,0 @@ -Restore elapsed time and unreachable object count in GC debug output. These -were inadvertently removed during a refactor of ``gc.c``. The debug log now -again reports elapsed collection time and the number of unreachable objects. -Contributed by Pål Grønås Drange. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-21-06-51-50.gh-issue-140406.0gJs8M.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-21-06-51-50.gh-issue-140406.0gJs8M.rst deleted file mode 100644 index 3506ba42581faa..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-21-06-51-50.gh-issue-140406.0gJs8M.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix memory leak when an object's :meth:`~object.__hash__` method returns an -object that isn't an :class:`int`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-21-09-20-03.gh-issue-140398.SoABwJ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-21-09-20-03.gh-issue-140398.SoABwJ.rst deleted file mode 100644 index 481dac7f26dd5e..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-21-09-20-03.gh-issue-140398.SoABwJ.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix memory leaks in :mod:`readline` functions -:func:`~readline.read_init_file`, :func:`~readline.read_history_file`, -:func:`~readline.write_history_file`, and -:func:`~readline.append_history_file` when :c:func:`PySys_Audit` fails. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-11-30-16.gh-issue-135904.3WE5oW.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-11-30-16.gh-issue-135904.3WE5oW.rst deleted file mode 100644 index b52a57dba4acae..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-11-30-16.gh-issue-135904.3WE5oW.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add special labels to the assembly created during stencil creation to -support relocations that the native object file format does not support. -Specifically, 19 bit branches for AArch64 in Mach-O object files. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-12-48-05.gh-issue-140476.F3-d1P.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-12-48-05.gh-issue-140476.F3-d1P.rst deleted file mode 100644 index a24033208c558c..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-12-48-05.gh-issue-140476.F3-d1P.rst +++ /dev/null @@ -1,2 +0,0 @@ -Optimize :c:func:`PySet_Add` for :class:`frozenset` in :term:`free threaded -` build. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-17-22-22.gh-issue-140431.m8D_A-.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-17-22-22.gh-issue-140431.m8D_A-.rst deleted file mode 100644 index 3d62d210f1f007..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-17-22-22.gh-issue-140431.m8D_A-.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a crash in Python's :term:`garbage collector ` due to -partially initialized :term:`coroutine` objects when coroutine origin tracking -depth is enabled (:func:`sys.set_coroutine_origin_tracking_depth`). diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-23-26-37.gh-issue-140443.wT5i1A.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-23-26-37.gh-issue-140443.wT5i1A.rst deleted file mode 100644 index a1fff8fef7ebe2..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-22-23-26-37.gh-issue-140443.wT5i1A.rst +++ /dev/null @@ -1,5 +0,0 @@ -The logarithm functions (such as :func:`math.log10` and :func:`math.log`) may now produce -slightly different results for extremely large integers that cannot be -converted to floats without overflow. These results are generally more -accurate, with reduced worst-case error and a tighter overall error -distribution. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-23-16-05-50.gh-issue-140471.Ax_aXn.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-23-16-05-50.gh-issue-140471.Ax_aXn.rst deleted file mode 100644 index afa9326fff3aee..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-23-16-05-50.gh-issue-140471.Ax_aXn.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix potential buffer overflow in :class:`ast.AST` node initialization when -encountering malformed :attr:`~ast.AST._fields` containing non-:class:`str`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-14-29-12.gh-issue-133467.A5d6TM.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-14-29-12.gh-issue-133467.A5d6TM.rst deleted file mode 100644 index f69786866e9878..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-14-29-12.gh-issue-133467.A5d6TM.rst +++ /dev/null @@ -1 +0,0 @@ -Fix race when updating :attr:`!type.__bases__` that could allow a read of :attr:`!type.__base__` to observe an inconsistent value on the free threaded build. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-16-42.gh-issue-140517.cqun-K.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-16-42.gh-issue-140517.cqun-K.rst deleted file mode 100644 index 15aaea8ab027e3..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-16-42.gh-issue-140517.cqun-K.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fixed a reference leak when iterating over the result of :func:`map` -with ``strict=True`` when the input iterables have different lengths. -Patch by Mikhail Efimov. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-42-33.gh-issue-140551.-9swrl.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-42-33.gh-issue-140551.-9swrl.rst deleted file mode 100644 index 8fd9b46c0aeabe..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-24-20-42-33.gh-issue-140551.-9swrl.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed crash in :class:`dict` if :meth:`dict.clear` is called at the lookup -stage. Patch by Mikhail Efimov and Inada Naoki. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-07-25-52.gh-issue-140544.lwjtQe.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-07-25-52.gh-issue-140544.lwjtQe.rst deleted file mode 100644 index 51d2b229ee5b80..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-07-25-52.gh-issue-140544.lwjtQe.rst +++ /dev/null @@ -1 +0,0 @@ -Speed up accessing interpreter state by caching it in a thread local variable. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst deleted file mode 100644 index 2c27525d9f782c..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-17-36-46.gh-issue-140576.kj0SCY.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed crash in :func:`tokenize.generate_tokens` in case of -specific incorrect input. Patch by Mikhail Efimov. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-21-31-43.gh-issue-131527.V-JVNP.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-21-31-43.gh-issue-131527.V-JVNP.rst deleted file mode 100644 index 9969ea058a3771..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-25-21-31-43.gh-issue-131527.V-JVNP.rst +++ /dev/null @@ -1,2 +0,0 @@ -Dynamic borrow checking for stackrefs is added to ``Py_STACKREF_DEBUG`` -mode. Patch by Mikhail Efimov. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-11-31-59.gh-issue-140729.t9JsNt.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-11-31-59.gh-issue-140729.t9JsNt.rst deleted file mode 100644 index 6725547667fb3c..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-11-31-59.gh-issue-140729.t9JsNt.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix pickling error in the sampling profiler when using ``concurrent.futures.ProcessPoolExecutor`` -script can not be properly pickled and executed in worker processes. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-20-59-10.gh-issue-140373.-uoaPP.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-20-59-10.gh-issue-140373.-uoaPP.rst deleted file mode 100644 index c9a97037920fda..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-29-20-59-10.gh-issue-140373.-uoaPP.rst +++ /dev/null @@ -1,2 +0,0 @@ -Correctly emit ``PY_UNWIND`` event when generator object is closed. Patch by -Mikhail Efimov. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-31-14-03-42.gh-issue-90344.gvZigO.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-31-14-03-42.gh-issue-90344.gvZigO.rst deleted file mode 100644 index b1d05354f65c71..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-31-14-03-42.gh-issue-90344.gvZigO.rst +++ /dev/null @@ -1 +0,0 @@ -Replace :class:`io.IncrementalNewlineDecoder` with non incremental newline decoders in codebase where :meth:`!io.IncrementalNewlineDecoder.decode` was being called once. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-12-47-38.gh-issue-140530.S934bp.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-12-47-38.gh-issue-140530.S934bp.rst deleted file mode 100644 index e3af493893afcb..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-12-47-38.gh-issue-140530.S934bp.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a reference leak when ``raise exc from cause`` fails. Patch by Bénédikt -Tran. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-15-28-33.gh-issue-140260.JNzlGz.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-15-28-33.gh-issue-140260.JNzlGz.rst deleted file mode 100644 index 96bf9b51e4862c..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-02-15-28-33.gh-issue-140260.JNzlGz.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :mod:`struct` data race in endian table initialization with -subinterpreters. Patch by Shamil Abdulaev. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-03-17-21-38.gh-issue-140939.FVboAw.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-03-17-21-38.gh-issue-140939.FVboAw.rst deleted file mode 100644 index a2921761f75556..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-03-17-21-38.gh-issue-140939.FVboAw.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix memory leak when :class:`bytearray` or :class:`bytes` is formated with the -``%*b`` format with a large width that results in a :exc:`MemoryError`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst deleted file mode 100644 index 0a615ed131127f..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-04-57-24.gh-issue-140479.lwQ2v2.rst +++ /dev/null @@ -1 +0,0 @@ -Update JIT compilation to use LLVM 21 at build time. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-12-18-06.gh-issue-140942.GYns6n.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-12-18-06.gh-issue-140942.GYns6n.rst deleted file mode 100644 index 20cfeca1e71dca..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-04-12-18-06.gh-issue-140942.GYns6n.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add ``.cjs`` to :mod:`mimetypes` to give CommonJS modules a MIME type of -``application/node``. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-05-19-50-37.gh-issue-140643.QCEOqG.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-05-19-50-37.gh-issue-140643.QCEOqG.rst deleted file mode 100644 index e1202dd1a17aec..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-05-19-50-37.gh-issue-140643.QCEOqG.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add support for ```` and ```` frames to -:mod:`!profiling.sampling` output to denote active garbage collection and -calls to native code. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-10-23-07-06.gh-issue-141312.H-58GB.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-10-23-07-06.gh-issue-141312.H-58GB.rst deleted file mode 100644 index fdb136cef3f33c..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-10-23-07-06.gh-issue-141312.H-58GB.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix the assertion failure in the ``__setstate__`` method of the range iterator -when a non-integer argument is passed. Patch by Sergey Miryanov. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-11-13-40-45.gh-issue-141367.I5KY7F.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-11-13-40-45.gh-issue-141367.I5KY7F.rst deleted file mode 100644 index cb830fcd9e1270..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-11-13-40-45.gh-issue-141367.I5KY7F.rst +++ /dev/null @@ -1,2 +0,0 @@ -Specialize ``CALL_LIST_APPEND`` instruction only for lists, not for list -subclasses, to avoid unnecessary deopt. Patch by Mikhail Efimov. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-00-19-45.gh-issue-141528.VWdax1.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-00-19-45.gh-issue-141528.VWdax1.rst deleted file mode 100644 index a51aa49522866b..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-00-19-45.gh-issue-141528.VWdax1.rst +++ /dev/null @@ -1,3 +0,0 @@ -Suggest using :meth:`concurrent.interpreters.Interpreter.close` instead of the -private ``_interpreters.destroy`` function when warning about remaining subinterpreters. -Patch by Sergey Miryanov. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-16-25-15.gh-issue-114203.n3tlQO.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-16-25-15.gh-issue-114203.n3tlQO.rst deleted file mode 100644 index 883f9333cae880..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-14-16-25-15.gh-issue-114203.n3tlQO.rst +++ /dev/null @@ -1 +0,0 @@ -Skip locking if object is already locked by two-mutex critical section. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-01-21-00.gh-issue-141579.aB7cD9.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-01-21-00.gh-issue-141579.aB7cD9.rst deleted file mode 100644 index 8ab9979c39917b..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-01-21-00.gh-issue-141579.aB7cD9.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :func:`sys.activate_stack_trampoline` to properly support the -``perf_jit`` backend. Patch by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Library/2019-06-02-13-56-16.gh-issue-81313.axawSH.rst b/Misc/NEWS.d/next/Library/2019-06-02-13-56-16.gh-issue-81313.axawSH.rst deleted file mode 100644 index 2291c938222062..00000000000000 --- a/Misc/NEWS.d/next/Library/2019-06-02-13-56-16.gh-issue-81313.axawSH.rst +++ /dev/null @@ -1 +0,0 @@ -Add the :mod:`math.integer` module (:pep:`791`). diff --git a/Misc/NEWS.d/next/Library/2023-03-21-10-59-40.gh-issue-102431.eUDnf4.rst b/Misc/NEWS.d/next/Library/2023-03-21-10-59-40.gh-issue-102431.eUDnf4.rst deleted file mode 100644 index e82ddb6e1011ad..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-21-10-59-40.gh-issue-102431.eUDnf4.rst +++ /dev/null @@ -1,2 +0,0 @@ -Clarify constraints for "logical" arguments in methods of -:class:`decimal.Context`. diff --git a/Misc/NEWS.d/next/Library/2024-05-28-17-14-30.gh-issue-119668.RrIGpn.rst b/Misc/NEWS.d/next/Library/2024-05-28-17-14-30.gh-issue-119668.RrIGpn.rst deleted file mode 100644 index 87cdf8d89d5202..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-05-28-17-14-30.gh-issue-119668.RrIGpn.rst +++ /dev/null @@ -1 +0,0 @@ -Publicly expose and document :class:`importlib.machinery.NamespacePath`. diff --git a/Misc/NEWS.d/next/Library/2024-06-26-16-16-43.gh-issue-121011.qW54eh.rst b/Misc/NEWS.d/next/Library/2024-06-26-16-16-43.gh-issue-121011.qW54eh.rst deleted file mode 100644 index aee7fe2bcb5c60..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-06-26-16-16-43.gh-issue-121011.qW54eh.rst +++ /dev/null @@ -1,2 +0,0 @@ -:func:`math.log` now supports arbitrary large integer-like arguments in the -same way as arbitrary large integer arguments. diff --git a/Misc/NEWS.d/next/Library/2024-08-08-12-39-36.gh-issue-122255.J_gU8Y.rst b/Misc/NEWS.d/next/Library/2024-08-08-12-39-36.gh-issue-122255.J_gU8Y.rst deleted file mode 100644 index 63e71c19f8b084..00000000000000 --- a/Misc/NEWS.d/next/Library/2024-08-08-12-39-36.gh-issue-122255.J_gU8Y.rst +++ /dev/null @@ -1,4 +0,0 @@ -In the :mod:`linecache` module and in the Python implementation of the -:mod:`warnings` module, a ``DeprecationWarning`` is issued when -``mod.__loader__`` differs from ``mod.__spec__.loader`` (like in the C -implementation of the :mod:`!warnings` module). diff --git a/Misc/NEWS.d/next/Library/2025-03-04-17-19-26.gh-issue-130693.Kv01r8.rst b/Misc/NEWS.d/next/Library/2025-03-04-17-19-26.gh-issue-130693.Kv01r8.rst deleted file mode 100644 index b175ab7cad468a..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-03-04-17-19-26.gh-issue-130693.Kv01r8.rst +++ /dev/null @@ -1 +0,0 @@ -Add support for ``-nolinestop``, and ``-strictlimits`` options to :meth:`!tkinter.Text.search`. Also add the :meth:`!tkinter.Text.search_all` method for ``-all`` and ``-overlap`` options. diff --git a/Misc/NEWS.d/next/Library/2025-03-12-18-57-10.gh-issue-131116.uTpwXZ.rst b/Misc/NEWS.d/next/Library/2025-03-12-18-57-10.gh-issue-131116.uTpwXZ.rst deleted file mode 100644 index f5e60ab6e8c4cb..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-03-12-18-57-10.gh-issue-131116.uTpwXZ.rst +++ /dev/null @@ -1,2 +0,0 @@ -:func:`inspect.getdoc` now correctly returns an inherited docstring on -:class:`~functools.cached_property` objects if none is given in a subclass. diff --git a/Misc/NEWS.d/next/Library/2025-04-18-18-08-05.gh-issue-132686.6kV_Gs.rst b/Misc/NEWS.d/next/Library/2025-04-18-18-08-05.gh-issue-132686.6kV_Gs.rst deleted file mode 100644 index d0c8e2d705cc73..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-04-18-18-08-05.gh-issue-132686.6kV_Gs.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add parameters *inherit_class_doc* and *fallback_to_class_doc* for -:func:`inspect.getdoc`. diff --git a/Misc/NEWS.d/next/Library/2025-05-07-22-09-28.gh-issue-133601.9kUL3P.rst b/Misc/NEWS.d/next/Library/2025-05-07-22-09-28.gh-issue-133601.9kUL3P.rst deleted file mode 100644 index 62f40aee7aaa4f..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-05-07-22-09-28.gh-issue-133601.9kUL3P.rst +++ /dev/null @@ -1 +0,0 @@ -Remove deprecated :func:`!typing.no_type_check_decorator`. diff --git a/Misc/NEWS.d/next/Library/2025-05-10-15-10-54.gh-issue-133789.I-ZlUX.rst b/Misc/NEWS.d/next/Library/2025-05-10-15-10-54.gh-issue-133789.I-ZlUX.rst deleted file mode 100644 index d2a4f7f42c3b38..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-05-10-15-10-54.gh-issue-133789.I-ZlUX.rst +++ /dev/null @@ -1 +0,0 @@ -Fix unpickling of :mod:`pathlib` objects that were pickled in Python 3.13. diff --git a/Misc/NEWS.d/next/Library/2025-06-10-18-02-29.gh-issue-135307.fXGrcK.rst b/Misc/NEWS.d/next/Library/2025-06-10-18-02-29.gh-issue-135307.fXGrcK.rst deleted file mode 100644 index 47e1feb5cbff09..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-06-10-18-02-29.gh-issue-135307.fXGrcK.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`email`: Fix exception in ``set_content()`` when encoding text -and max_line_length is set to ``0`` or ``None`` (unlimited). diff --git a/Misc/NEWS.d/next/Library/2025-06-29-22-01-00.gh-issue-133390.I1DW_3.rst b/Misc/NEWS.d/next/Library/2025-06-29-22-01-00.gh-issue-133390.I1DW_3.rst deleted file mode 100644 index c57f802d4c8a78..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-06-29-22-01-00.gh-issue-133390.I1DW_3.rst +++ /dev/null @@ -1,2 +0,0 @@ -Support table, index, trigger, view, column, function, and schema completion -for :mod:`sqlite3`'s :ref:`command-line interface `. diff --git a/Misc/NEWS.d/next/Library/2025-07-01-04-57-57.gh-issue-136057.4-t596.rst b/Misc/NEWS.d/next/Library/2025-07-01-04-57-57.gh-issue-136057.4-t596.rst deleted file mode 100644 index e237a0e98cc486..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-07-01-04-57-57.gh-issue-136057.4-t596.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed the bug in :mod:`pdb` and :mod:`bdb` where ``next`` and ``step`` can't go over the line if a loop exists in the line. diff --git a/Misc/NEWS.d/next/Library/2025-07-14-09-33-17.gh-issue-55531.Gt2e12.rst b/Misc/NEWS.d/next/Library/2025-07-14-09-33-17.gh-issue-55531.Gt2e12.rst deleted file mode 100644 index 70e39a4f2c167c..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-07-14-09-33-17.gh-issue-55531.Gt2e12.rst +++ /dev/null @@ -1,4 +0,0 @@ -:mod:`encodings`: Improve :func:`~encodings.normalize_encoding` performance -by implementing the function in C using the private -``_Py_normalize_encoding`` which has been modified to make lowercase -conversion optional. diff --git a/Misc/NEWS.d/next/Library/2025-08-11-04-52-18.gh-issue-137627.Ku5Yi2.rst b/Misc/NEWS.d/next/Library/2025-08-11-04-52-18.gh-issue-137627.Ku5Yi2.rst deleted file mode 100644 index 855070ed6f4511..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-08-11-04-52-18.gh-issue-137627.Ku5Yi2.rst +++ /dev/null @@ -1 +0,0 @@ -Speed up :meth:`csv.Sniffer.sniff` delimiter detection by up to 1.6x. diff --git a/Misc/NEWS.d/next/Library/2025-08-15-20-35-30.gh-issue-69528.qc-Eh_.rst b/Misc/NEWS.d/next/Library/2025-08-15-20-35-30.gh-issue-69528.qc-Eh_.rst deleted file mode 100644 index b18781e0dceb8c..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-08-15-20-35-30.gh-issue-69528.qc-Eh_.rst +++ /dev/null @@ -1,2 +0,0 @@ -The :attr:`~io.FileIO.mode` attribute of files opened in the ``'wb+'`` mode is -now ``'wb+'`` instead of ``'rb+'``. diff --git a/Misc/NEWS.d/next/Library/2025-08-26-08-17-56.gh-issue-138151.I6CdAk.rst b/Misc/NEWS.d/next/Library/2025-08-26-08-17-56.gh-issue-138151.I6CdAk.rst deleted file mode 100644 index de29f536afc95e..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-08-26-08-17-56.gh-issue-138151.I6CdAk.rst +++ /dev/null @@ -1,3 +0,0 @@ -In :mod:`annotationlib`, improve evaluation of forward references to -nonlocal variables that are not yet defined when the annotations are -initially evaluated. diff --git a/Misc/NEWS.d/next/Library/2025-09-03-18-26-07.gh-issue-138425.cVE9Ho.rst b/Misc/NEWS.d/next/Library/2025-09-03-18-26-07.gh-issue-138425.cVE9Ho.rst deleted file mode 100644 index 328e5988cb0b51..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-09-03-18-26-07.gh-issue-138425.cVE9Ho.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix partial evaluation of :class:`annotationlib.ForwardRef` objects which rely -on names defined as globals. diff --git a/Misc/NEWS.d/next/Library/2025-09-03-20-18-39.gh-issue-98896.tjez89.rst b/Misc/NEWS.d/next/Library/2025-09-03-20-18-39.gh-issue-98896.tjez89.rst deleted file mode 100644 index 6831499c0afb43..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-09-03-20-18-39.gh-issue-98896.tjez89.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a failure in multiprocessing resource_tracker when SharedMemory names contain colons. -Patch by Rani Pinchuk. diff --git a/Misc/NEWS.d/next/Library/2025-09-11-15-03-37.gh-issue-138775.w7rnSx.rst b/Misc/NEWS.d/next/Library/2025-09-11-15-03-37.gh-issue-138775.w7rnSx.rst deleted file mode 100644 index 455c1a9925a5e1..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-09-11-15-03-37.gh-issue-138775.w7rnSx.rst +++ /dev/null @@ -1,2 +0,0 @@ -Use of ``python -m`` with :mod:`base64` has been fixed to detect input from a -terminal so that it properly notices EOF. diff --git a/Misc/NEWS.d/next/Library/2025-09-12-09-34-37.gh-issue-138764.mokHoY.rst b/Misc/NEWS.d/next/Library/2025-09-12-09-34-37.gh-issue-138764.mokHoY.rst deleted file mode 100644 index 85ebef8ff11d5c..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-09-12-09-34-37.gh-issue-138764.mokHoY.rst +++ /dev/null @@ -1,3 +0,0 @@ -Prevent :func:`annotationlib.call_annotate_function` from calling ``__annotate__`` functions that don't support ``VALUE_WITH_FAKE_GLOBALS`` in a fake globals namespace with empty globals. - -Make ``FORWARDREF`` and ``STRING`` annotations fall back to using ``VALUE`` annotations in the case that neither their own format, nor ``VALUE_WITH_FAKE_GLOBALS`` are supported. diff --git a/Misc/NEWS.d/next/Library/2025-09-13-12-19-17.gh-issue-138859.PxjIoN.rst b/Misc/NEWS.d/next/Library/2025-09-13-12-19-17.gh-issue-138859.PxjIoN.rst deleted file mode 100644 index a5d4dd042fcd5b..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-09-13-12-19-17.gh-issue-138859.PxjIoN.rst +++ /dev/null @@ -1 +0,0 @@ -Fix generic type parameterization raising a :exc:`TypeError` when omitting a :class:`ParamSpec` that has a default which is not a list of types. diff --git a/Misc/NEWS.d/next/Library/2025-09-15-21-03-11.gh-issue-138891.oZFdtR.rst b/Misc/NEWS.d/next/Library/2025-09-15-21-03-11.gh-issue-138891.oZFdtR.rst deleted file mode 100644 index f7ecb05d20c241..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-09-15-21-03-11.gh-issue-138891.oZFdtR.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix ``SyntaxError`` when ``inspect.get_annotations(f, eval_str=True)`` is -called on a function annotated with a :pep:`646` ``star_expression`` diff --git a/Misc/NEWS.d/next/Library/2025-09-18-21-25-41.gh-issue-83714.TQjDWZ.rst b/Misc/NEWS.d/next/Library/2025-09-18-21-25-41.gh-issue-83714.TQjDWZ.rst deleted file mode 100644 index 3653eb9a114a35..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-09-18-21-25-41.gh-issue-83714.TQjDWZ.rst +++ /dev/null @@ -1,2 +0,0 @@ -Implement :func:`os.statx` on Linux kernel versions 4.11 and later with glibc -versions 2.28 and later. Contributed by Jeffrey Bosboom and Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2025-09-23-09-46-46.gh-issue-139246.pzfM-w.rst b/Misc/NEWS.d/next/Library/2025-09-23-09-46-46.gh-issue-139246.pzfM-w.rst deleted file mode 100644 index a816bda5cfe8e8..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-09-23-09-46-46.gh-issue-139246.pzfM-w.rst +++ /dev/null @@ -1 +0,0 @@ -fix: paste zero-width in default repl width is wrong. diff --git a/Misc/NEWS.d/next/Library/2025-09-25-20-16-10.gh-issue-101828.yTxJlJ.rst b/Misc/NEWS.d/next/Library/2025-09-25-20-16-10.gh-issue-101828.yTxJlJ.rst deleted file mode 100644 index 1d100180c072ec..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-09-25-20-16-10.gh-issue-101828.yTxJlJ.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix ``'shift_jisx0213'``, ``'shift_jis_2004'``, ``'euc_jisx0213'`` and -``'euc_jis_2004'`` codecs truncating null chars -as they were treated as part of multi-character sequences. diff --git a/Misc/NEWS.d/next/Library/2025-09-30-12-52-54.gh-issue-63161.mECM1A.rst b/Misc/NEWS.d/next/Library/2025-09-30-12-52-54.gh-issue-63161.mECM1A.rst deleted file mode 100644 index 3daed20d099a8a..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-09-30-12-52-54.gh-issue-63161.mECM1A.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix :func:`tokenize.detect_encoding`. Support non-UTF-8 shebang and comments -if non-UTF-8 encoding is specified. Detect decoding error for non-UTF-8 -encoding. Detect null bytes in source code. diff --git a/Misc/NEWS.d/next/Library/2025-10-02-22-29-00.gh-issue-139462.VZXUHe.rst b/Misc/NEWS.d/next/Library/2025-10-02-22-29-00.gh-issue-139462.VZXUHe.rst deleted file mode 100644 index 390a6124386151..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-02-22-29-00.gh-issue-139462.VZXUHe.rst +++ /dev/null @@ -1,3 +0,0 @@ -When a child process in a :class:`concurrent.futures.ProcessPoolExecutor` -terminates abruptly, the resulting traceback will now tell you the PID -and exit code of the terminated process. Contributed by Jonathan Berg. diff --git a/Misc/NEWS.d/next/Library/2025-10-11-09-07-06.gh-issue-139940.g54efZ.rst b/Misc/NEWS.d/next/Library/2025-10-11-09-07-06.gh-issue-139940.g54efZ.rst deleted file mode 100644 index 2501135e657e7d..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-11-09-07-06.gh-issue-139940.g54efZ.rst +++ /dev/null @@ -1 +0,0 @@ -Print clearer error message when using ``pdb`` to attach to a non-existing process. diff --git a/Misc/NEWS.d/next/Library/2025-10-13-11-25-41.gh-issue-136702.uvLGK1.rst b/Misc/NEWS.d/next/Library/2025-10-13-11-25-41.gh-issue-136702.uvLGK1.rst deleted file mode 100644 index 88303f017f58c4..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-13-11-25-41.gh-issue-136702.uvLGK1.rst +++ /dev/null @@ -1,3 +0,0 @@ -:mod:`encodings`: Deprecate passing a non-ascii *encoding* name to -:func:`encodings.normalize_encoding` and schedule removal of support for -Python 3.17. diff --git a/Misc/NEWS.d/next/Library/2025-10-14-20-27-06.gh-issue-76007.2NcUbo.rst b/Misc/NEWS.d/next/Library/2025-10-14-20-27-06.gh-issue-76007.2NcUbo.rst deleted file mode 100644 index 567fb5ef90475c..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-14-20-27-06.gh-issue-76007.2NcUbo.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`zlib`: Deprecate ``__version__`` and schedule for removal in Python -3.20. diff --git a/Misc/NEWS.d/next/Library/2025-10-15-02-26-50.gh-issue-140135.54JYfM.rst b/Misc/NEWS.d/next/Library/2025-10-15-02-26-50.gh-issue-140135.54JYfM.rst deleted file mode 100644 index 8d5a76af90906a..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-15-02-26-50.gh-issue-140135.54JYfM.rst +++ /dev/null @@ -1,2 +0,0 @@ -Speed up :meth:`io.RawIOBase.readall` by using PyBytesWriter API (about 4x -faster) diff --git a/Misc/NEWS.d/next/Library/2025-10-15-15-10-34.gh-issue-140166.NtxRez.rst b/Misc/NEWS.d/next/Library/2025-10-15-15-10-34.gh-issue-140166.NtxRez.rst deleted file mode 100644 index c140db9dcd5150..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-15-15-10-34.gh-issue-140166.NtxRez.rst +++ /dev/null @@ -1 +0,0 @@ -:mod:`mimetypes`: Per the `IANA assignment `_, update the MIME type for the ``.texi`` and ``.texinfo`` file formats to ``application/texinfo``, instead of ``application/x-texinfo``. diff --git a/Misc/NEWS.d/next/Library/2025-10-15-17-23-51.gh-issue-140141.j2mUDB.rst b/Misc/NEWS.d/next/Library/2025-10-15-17-23-51.gh-issue-140141.j2mUDB.rst deleted file mode 100644 index 2edadbc3e3887c..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-15-17-23-51.gh-issue-140141.j2mUDB.rst +++ /dev/null @@ -1,5 +0,0 @@ -The :py:class:`importlib.metadata.PackageNotFoundError` traceback raised when -``importlib.metadata.Distribution.from_name`` cannot discover a -distribution no longer includes a transient :exc:`StopIteration` exception trace. - -Contributed by Bartosz Sławecki in :gh:`140142`. diff --git a/Misc/NEWS.d/next/Library/2025-10-15-20-47-04.gh-issue-140120.3gffZq.rst b/Misc/NEWS.d/next/Library/2025-10-15-20-47-04.gh-issue-140120.3gffZq.rst deleted file mode 100644 index 9eefe1405203bd..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-15-20-47-04.gh-issue-140120.3gffZq.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed a memory leak in :mod:`hmac` when it was using the hacl-star backend. -Discovered by ``@ashm-dev`` using AddressSanitizer. diff --git a/Misc/NEWS.d/next/Library/2025-10-15-21-42-13.gh-issue-140041._Fka2j.rst b/Misc/NEWS.d/next/Library/2025-10-15-21-42-13.gh-issue-140041._Fka2j.rst deleted file mode 100644 index 243ff39311cf06..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-15-21-42-13.gh-issue-140041._Fka2j.rst +++ /dev/null @@ -1 +0,0 @@ -Fix import of :mod:`ctypes` on Android and Cygwin when ABI flags are present. diff --git a/Misc/NEWS.d/next/Library/2025-10-16-16-10-11.gh-issue-139707.zR6Qtn.rst b/Misc/NEWS.d/next/Library/2025-10-16-16-10-11.gh-issue-139707.zR6Qtn.rst deleted file mode 100644 index c5460aae8b3638..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-16-16-10-11.gh-issue-139707.zR6Qtn.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve :exc:`ModuleNotFoundError` error message when a :term:`standard library` -module is missing. diff --git a/Misc/NEWS.d/next/Library/2025-10-16-17-17-20.gh-issue-135801.faH3fa.rst b/Misc/NEWS.d/next/Library/2025-10-16-17-17-20.gh-issue-135801.faH3fa.rst deleted file mode 100644 index d680312d5829fb..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-16-17-17-20.gh-issue-135801.faH3fa.rst +++ /dev/null @@ -1,6 +0,0 @@ -Improve filtering by module in :func:`warnings.warn_explicit` if no *module* -argument is passed. It now tests the module regular expression in the -warnings filter not only against the filename with ``.py`` stripped, but -also against module names constructed starting from different parent -directories of the filename (with ``/__init__.py``, ``.py`` and, on Windows, -``.pyw`` stripped). diff --git a/Misc/NEWS.d/next/Library/2025-10-16-22-49-16.gh-issue-140212.llBNd0.rst b/Misc/NEWS.d/next/Library/2025-10-16-22-49-16.gh-issue-140212.llBNd0.rst deleted file mode 100644 index 5563d07717117e..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-16-22-49-16.gh-issue-140212.llBNd0.rst +++ /dev/null @@ -1,5 +0,0 @@ -Calendar's HTML formatting now accepts year and month as options. -Previously, running ``python -m calendar -t html 2025 10`` would result in an -error message. It now generates an HTML document displaying the calendar for -the specified month. -Contributed by Pål Grønås Drange. diff --git a/Misc/NEWS.d/next/Library/2025-10-17-12-33-01.gh-issue-140251.esM-OX.rst b/Misc/NEWS.d/next/Library/2025-10-17-12-33-01.gh-issue-140251.esM-OX.rst deleted file mode 100644 index cb08e02429b229..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-17-12-33-01.gh-issue-140251.esM-OX.rst +++ /dev/null @@ -1 +0,0 @@ -Colorize the default import statement ``import asyncio`` in asyncio REPL. diff --git a/Misc/NEWS.d/next/Library/2025-10-17-20-42-38.gh-issue-129117.X9jr4p.rst b/Misc/NEWS.d/next/Library/2025-10-17-20-42-38.gh-issue-129117.X9jr4p.rst deleted file mode 100644 index 8767b1bb4837ad..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-17-20-42-38.gh-issue-129117.X9jr4p.rst +++ /dev/null @@ -1,3 +0,0 @@ -:mod:`unicodedata`: Add :func:`~unicodedata.isxidstart` and -:func:`~unicodedata.isxidcontinue` functions to check whether a character can -start or continue a `Unicode Standard Annex #31 `_ identifier. diff --git a/Misc/NEWS.d/next/Library/2025-10-17-23-58-11.gh-issue-140272.lhY8uS.rst b/Misc/NEWS.d/next/Library/2025-10-17-23-58-11.gh-issue-140272.lhY8uS.rst deleted file mode 100644 index 666a45055f5a58..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-17-23-58-11.gh-issue-140272.lhY8uS.rst +++ /dev/null @@ -1 +0,0 @@ -Fix memory leak in the :meth:`!clear` method of the :mod:`dbm.gnu` database. diff --git a/Misc/NEWS.d/next/Library/2025-10-18-14-30-21.gh-issue-76007.peEgcr.rst b/Misc/NEWS.d/next/Library/2025-10-18-14-30-21.gh-issue-76007.peEgcr.rst deleted file mode 100644 index be56b2ca6a1c97..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-18-14-30-21.gh-issue-76007.peEgcr.rst +++ /dev/null @@ -1 +0,0 @@ -Deprecate ``__version__`` from a :mod:`imaplib`. Patch by Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2025-10-18-15-20-25.gh-issue-76007.SNUzRq.rst b/Misc/NEWS.d/next/Library/2025-10-18-15-20-25.gh-issue-76007.SNUzRq.rst deleted file mode 100644 index 6a91fc41b0ab0c..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-18-15-20-25.gh-issue-76007.SNUzRq.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`decimal`: Deprecate ``__version__`` and replace with -:data:`decimal.SPEC_VERSION`. diff --git a/Misc/NEWS.d/next/Library/2025-10-20-12-33-49.gh-issue-140348.SAKnQZ.rst b/Misc/NEWS.d/next/Library/2025-10-20-12-33-49.gh-issue-140348.SAKnQZ.rst deleted file mode 100644 index 16d5b2a8bf03d0..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-20-12-33-49.gh-issue-140348.SAKnQZ.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix regression in Python 3.14.0 where using the ``|`` operator on a -:class:`typing.Union` object combined with an object that is not a type -would raise an error. diff --git a/Misc/NEWS.d/next/Library/2025-10-21-15-54-13.gh-issue-137530.ZyIVUH.rst b/Misc/NEWS.d/next/Library/2025-10-21-15-54-13.gh-issue-137530.ZyIVUH.rst deleted file mode 100644 index 4ff55b41dea96e..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-21-15-54-13.gh-issue-137530.ZyIVUH.rst +++ /dev/null @@ -1 +0,0 @@ -:mod:`dataclasses` Fix annotations for generated ``__init__`` methods by replacing the annotations that were in-line in the generated source code with ``__annotate__`` functions attached to the methods. diff --git a/Misc/NEWS.d/next/Library/2025-10-22-12-56-57.gh-issue-140448.GsEkXD.rst b/Misc/NEWS.d/next/Library/2025-10-22-12-56-57.gh-issue-140448.GsEkXD.rst deleted file mode 100644 index db7f92e136d41b..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-22-12-56-57.gh-issue-140448.GsEkXD.rst +++ /dev/null @@ -1,2 +0,0 @@ -Change the default of ``suggest_on_error`` to ``True`` in -``argparse.ArgumentParser``. diff --git a/Misc/NEWS.d/next/Library/2025-10-22-20-52-13.gh-issue-140474.xIWlip.rst b/Misc/NEWS.d/next/Library/2025-10-22-20-52-13.gh-issue-140474.xIWlip.rst deleted file mode 100644 index aca4e68b1e5e49..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-22-20-52-13.gh-issue-140474.xIWlip.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix memory leak in :class:`array.array` when creating arrays from an empty -:class:`str` and the ``u`` type code. diff --git a/Misc/NEWS.d/next/Library/2025-10-23-12-12-22.gh-issue-138774.mnh2gU.rst b/Misc/NEWS.d/next/Library/2025-10-23-12-12-22.gh-issue-138774.mnh2gU.rst deleted file mode 100644 index e12f789e674454..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-23-12-12-22.gh-issue-138774.mnh2gU.rst +++ /dev/null @@ -1,2 +0,0 @@ -:func:`ast.unparse` now generates full source code when handling -:class:`ast.Interpolation` nodes that do not have a specified source. diff --git a/Misc/NEWS.d/next/Library/2025-10-23-13-42-15.gh-issue-140481.XKxWpq.rst b/Misc/NEWS.d/next/Library/2025-10-23-13-42-15.gh-issue-140481.XKxWpq.rst deleted file mode 100644 index 1f511c3b9d0583..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-23-13-42-15.gh-issue-140481.XKxWpq.rst +++ /dev/null @@ -1 +0,0 @@ -Improve error message when trying to iterate a Tk widget, image or font. diff --git a/Misc/NEWS.d/next/Library/2025-10-23-19-39-16.gh-issue-138162.Znw5DN.rst b/Misc/NEWS.d/next/Library/2025-10-23-19-39-16.gh-issue-138162.Znw5DN.rst deleted file mode 100644 index ef7a90bc37e650..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-23-19-39-16.gh-issue-138162.Znw5DN.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :class:`logging.LoggerAdapter` with ``merge_extra=True`` and without the -*extra* argument. diff --git a/Misc/NEWS.d/next/Library/2025-10-25-21-04-00.gh-issue-140607.oOZGxS.rst b/Misc/NEWS.d/next/Library/2025-10-25-21-04-00.gh-issue-140607.oOZGxS.rst deleted file mode 100644 index cc33217c9f563e..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-25-21-04-00.gh-issue-140607.oOZGxS.rst +++ /dev/null @@ -1,2 +0,0 @@ -Inside :meth:`io.RawIOBase.read`, validate that the count of bytes returned by -:meth:`io.RawIOBase.readinto` is valid (inside the provided buffer). diff --git a/Misc/NEWS.d/next/Library/2025-10-25-21-26-16.gh-issue-140593.OxlLc9.rst b/Misc/NEWS.d/next/Library/2025-10-25-21-26-16.gh-issue-140593.OxlLc9.rst deleted file mode 100644 index 612ad82dc64309..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-25-21-26-16.gh-issue-140593.OxlLc9.rst +++ /dev/null @@ -1,3 +0,0 @@ -:mod:`xml.parsers.expat`: Fix a memory leak that could affect users with -:meth:`~xml.parsers.expat.xmlparser.ElementDeclHandler` set to a custom -element declaration handler. Patch by Sebastian Pipping. diff --git a/Misc/NEWS.d/next/Library/2025-10-25-22-55-07.gh-issue-140601.In3MlS.rst b/Misc/NEWS.d/next/Library/2025-10-25-22-55-07.gh-issue-140601.In3MlS.rst deleted file mode 100644 index 72666bb8224d63..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-25-22-55-07.gh-issue-140601.In3MlS.rst +++ /dev/null @@ -1,4 +0,0 @@ -:func:`xml.etree.ElementTree.iterparse` now emits a :exc:`ResourceWarning` -when the iterator is not explicitly closed and was opened with a filename. -This helps developers identify and fix resource leaks. Patch by Osama -Abdelkader. diff --git a/Misc/NEWS.d/next/Library/2025-10-26-16-24-12.gh-issue-140633.ioayC1.rst b/Misc/NEWS.d/next/Library/2025-10-26-16-24-12.gh-issue-140633.ioayC1.rst deleted file mode 100644 index 9675a5d427a0d9..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-26-16-24-12.gh-issue-140633.ioayC1.rst +++ /dev/null @@ -1,2 +0,0 @@ -Ignore :exc:`AttributeError` when setting a module's ``__file__`` attribute -when loading an extension module packaged as Apple Framework. diff --git a/Misc/NEWS.d/next/Library/2025-10-27-00-40-49.gh-issue-140650.DYJPJ9.rst b/Misc/NEWS.d/next/Library/2025-10-27-00-40-49.gh-issue-140650.DYJPJ9.rst deleted file mode 100644 index 2ae153a64808e8..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-27-00-40-49.gh-issue-140650.DYJPJ9.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix an issue where closing :class:`io.BufferedWriter` could crash if -the closed attribute raised an exception on access or could not be -converted to a boolean. diff --git a/Misc/NEWS.d/next/Library/2025-10-27-13-49-31.gh-issue-140634.ULng9G.rst b/Misc/NEWS.d/next/Library/2025-10-27-13-49-31.gh-issue-140634.ULng9G.rst deleted file mode 100644 index b1ba9b26ad5431..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-27-13-49-31.gh-issue-140634.ULng9G.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a reference counting bug in :meth:`!os.sched_param.__reduce__`. diff --git a/Misc/NEWS.d/next/Library/2025-10-27-16-01-41.gh-issue-125434.qy0uRA.rst b/Misc/NEWS.d/next/Library/2025-10-27-16-01-41.gh-issue-125434.qy0uRA.rst deleted file mode 100644 index 299e9f04df7c39..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-27-16-01-41.gh-issue-125434.qy0uRA.rst +++ /dev/null @@ -1,2 +0,0 @@ -Display thread name in :mod:`faulthandler` on Windows. Patch by Victor -Stinner. diff --git a/Misc/NEWS.d/next/Library/2025-10-27-18-29-42.gh-issue-140590.LT9HHn.rst b/Misc/NEWS.d/next/Library/2025-10-27-18-29-42.gh-issue-140590.LT9HHn.rst deleted file mode 100644 index 802183673cfacc..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-27-18-29-42.gh-issue-140590.LT9HHn.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix arguments checking for the :meth:`!functools.partial.__setstate__` that -may lead to internal state corruption and crash. Patch by Sergey Miryanov. diff --git a/Misc/NEWS.d/next/Library/2025-10-28-02-46-56.gh-issue-139946.aN3_uY.rst b/Misc/NEWS.d/next/Library/2025-10-28-02-46-56.gh-issue-139946.aN3_uY.rst deleted file mode 100644 index 4c68d4cd94bf78..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-28-02-46-56.gh-issue-139946.aN3_uY.rst +++ /dev/null @@ -1 +0,0 @@ -Error and warning keywords in ``argparse.ArgumentParser`` messages are now colorized when color output is enabled, fixing a visual inconsistency in which they remained plain text while other output was colorized. diff --git a/Misc/NEWS.d/next/Library/2025-10-28-17-43-51.gh-issue-140228.8kfHhO.rst b/Misc/NEWS.d/next/Library/2025-10-28-17-43-51.gh-issue-140228.8kfHhO.rst deleted file mode 100644 index b3b692bae62c5d..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-28-17-43-51.gh-issue-140228.8kfHhO.rst +++ /dev/null @@ -1 +0,0 @@ -Avoid making unnecessary filesystem calls for frozen modules in :mod:`linecache` when the global module cache is not present. diff --git a/Misc/NEWS.d/next/Library/2025-10-29-09-40-10.gh-issue-140741.L13UCV.rst b/Misc/NEWS.d/next/Library/2025-10-29-09-40-10.gh-issue-140741.L13UCV.rst deleted file mode 100644 index 9fa8c561a03c78..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-29-09-40-10.gh-issue-140741.L13UCV.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :func:`profiling.sampling.sample` incorrectly handling a -:exc:`FileNotFoundError` or :exc:`PermissionError`. diff --git a/Misc/NEWS.d/next/Library/2025-10-29-16-12-41.gh-issue-120057.qGj5Dl.rst b/Misc/NEWS.d/next/Library/2025-10-29-16-12-41.gh-issue-120057.qGj5Dl.rst deleted file mode 100644 index f6b42be1fbf50d..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-29-16-12-41.gh-issue-120057.qGj5Dl.rst +++ /dev/null @@ -1 +0,0 @@ -Add :func:`os.reload_environ` to ``os.__all__``. diff --git a/Misc/NEWS.d/next/Library/2025-10-29-16-53-00.gh-issue-140766.CNagKF.rst b/Misc/NEWS.d/next/Library/2025-10-29-16-53-00.gh-issue-140766.CNagKF.rst deleted file mode 100644 index fce8dd33757aae..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-29-16-53-00.gh-issue-140766.CNagKF.rst +++ /dev/null @@ -1 +0,0 @@ -Add :func:`enum.show_flag_values` and ``enum.bin`` to ``enum.__all__``. diff --git a/Misc/NEWS.d/next/Library/2025-10-30-12-36-19.gh-issue-140790._3T6-N.rst b/Misc/NEWS.d/next/Library/2025-10-30-12-36-19.gh-issue-140790._3T6-N.rst deleted file mode 100644 index 03856f0b9b6d0a..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-30-12-36-19.gh-issue-140790._3T6-N.rst +++ /dev/null @@ -1 +0,0 @@ -Initialize all Pdb's instance variables in ``__init__``, remove some hasattr/getattr diff --git a/Misc/NEWS.d/next/Library/2025-10-30-15-33-07.gh-issue-137821.8_Iavt.rst b/Misc/NEWS.d/next/Library/2025-10-30-15-33-07.gh-issue-137821.8_Iavt.rst deleted file mode 100644 index 7ccbfc3cb950bf..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-30-15-33-07.gh-issue-137821.8_Iavt.rst +++ /dev/null @@ -1,2 +0,0 @@ -Convert ``_json`` module to use Argument Clinic. -Patched by Yoonho Hann. diff --git a/Misc/NEWS.d/next/Library/2025-10-31-13-57-55.gh-issue-103847.VM7TnW.rst b/Misc/NEWS.d/next/Library/2025-10-31-13-57-55.gh-issue-103847.VM7TnW.rst deleted file mode 100644 index e14af7d97083d6..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-31-13-57-55.gh-issue-103847.VM7TnW.rst +++ /dev/null @@ -1 +0,0 @@ -Fix hang when cancelling process created by :func:`asyncio.create_subprocess_exec` or :func:`asyncio.create_subprocess_shell`. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Library/2025-10-31-15-06-26.gh-issue-140691.JzHGtg.rst b/Misc/NEWS.d/next/Library/2025-10-31-15-06-26.gh-issue-140691.JzHGtg.rst deleted file mode 100644 index 84b6195c9262c8..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-31-15-06-26.gh-issue-140691.JzHGtg.rst +++ /dev/null @@ -1,3 +0,0 @@ -In :mod:`urllib.request`, when opening a FTP URL fails because a data -connection cannot be made, the control connection's socket is now closed to -avoid a :exc:`ResourceWarning`. diff --git a/Misc/NEWS.d/next/Library/2025-10-31-16-25-13.gh-issue-140808.XBiQ4j.rst b/Misc/NEWS.d/next/Library/2025-10-31-16-25-13.gh-issue-140808.XBiQ4j.rst deleted file mode 100644 index 090f39c6e25fdf..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-31-16-25-13.gh-issue-140808.XBiQ4j.rst +++ /dev/null @@ -1 +0,0 @@ -The internal class ``mailbox._ProxyFile`` is no longer a parameterized generic. diff --git a/Misc/NEWS.d/next/Library/2025-11-01-00-34-53.gh-issue-140826.JEDd7U.rst b/Misc/NEWS.d/next/Library/2025-11-01-00-34-53.gh-issue-140826.JEDd7U.rst deleted file mode 100644 index 875d15f2f8917c..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-11-01-00-34-53.gh-issue-140826.JEDd7U.rst +++ /dev/null @@ -1,2 +0,0 @@ -Now :class:`!winreg.HKEYType` objects are compared by their underlying Windows -registry handle value instead of their object identity. diff --git a/Misc/NEWS.d/next/Library/2025-11-01-00-36-14.gh-issue-140874.eAWt3K.rst b/Misc/NEWS.d/next/Library/2025-11-01-00-36-14.gh-issue-140874.eAWt3K.rst deleted file mode 100644 index a48162de76b496..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-11-01-00-36-14.gh-issue-140874.eAWt3K.rst +++ /dev/null @@ -1 +0,0 @@ -Bump the version of pip bundled in ensurepip to version 25.3 diff --git a/Misc/NEWS.d/next/Library/2025-11-01-14-44-09.gh-issue-140873.kfuc9B.rst b/Misc/NEWS.d/next/Library/2025-11-01-14-44-09.gh-issue-140873.kfuc9B.rst deleted file mode 100644 index e15057640646d6..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-11-01-14-44-09.gh-issue-140873.kfuc9B.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add support of non-:term:`descriptor` callables in -:func:`functools.singledispatchmethod`. diff --git a/Misc/NEWS.d/next/Library/2025-11-02-09-37-22.gh-issue-140734.f8gST9.rst b/Misc/NEWS.d/next/Library/2025-11-02-09-37-22.gh-issue-140734.f8gST9.rst deleted file mode 100644 index 46582f7fcf417c..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-11-02-09-37-22.gh-issue-140734.f8gST9.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`multiprocessing`: fix off-by-one error when checking the length -of a temporary socket file path. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2025-11-02-11-46-00.gh-issue-100218.9Ezfdq.rst b/Misc/NEWS.d/next/Library/2025-11-02-11-46-00.gh-issue-100218.9Ezfdq.rst deleted file mode 100644 index 2f7500d295578b..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-11-02-11-46-00.gh-issue-100218.9Ezfdq.rst +++ /dev/null @@ -1,3 +0,0 @@ -Correctly set :attr:`~OSError.errno` when :func:`socket.if_nametoindex` or -:func:`socket.if_indextoname` raise an :exc:`OSError`. Patch by Bénédikt -Tran. diff --git a/Misc/NEWS.d/next/Library/2025-11-02-19-23-32.gh-issue-140815.McEG-T.rst b/Misc/NEWS.d/next/Library/2025-11-02-19-23-32.gh-issue-140815.McEG-T.rst deleted file mode 100644 index 18c4d3836efef1..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-11-02-19-23-32.gh-issue-140815.McEG-T.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`faulthandler` now detects if a frame or a code object is invalid or -freed. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2025-11-03-05-38-31.gh-issue-125115.jGS8MN.rst b/Misc/NEWS.d/next/Library/2025-11-03-05-38-31.gh-issue-125115.jGS8MN.rst deleted file mode 100644 index d36debec3ed6cc..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-11-03-05-38-31.gh-issue-125115.jGS8MN.rst +++ /dev/null @@ -1 +0,0 @@ -Refactor the :mod:`pdb` parsing issue so positional arguments can pass through intuitively. diff --git a/Misc/NEWS.d/next/Library/2025-11-03-16-23-54.gh-issue-140797.DuFEeR.rst b/Misc/NEWS.d/next/Library/2025-11-03-16-23-54.gh-issue-140797.DuFEeR.rst deleted file mode 100644 index 493b740261e64c..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-11-03-16-23-54.gh-issue-140797.DuFEeR.rst +++ /dev/null @@ -1,2 +0,0 @@ -The undocumented :class:`!re.Scanner` class now forbids regular expressions containing capturing groups in its lexicon patterns. Patterns using capturing groups could -previously lead to crashes with segmentation fault. Use non-capturing groups (?:...) instead. diff --git a/Misc/NEWS.d/next/Library/2025-11-04-12-16-13.gh-issue-75593.EFVhKR.rst b/Misc/NEWS.d/next/Library/2025-11-04-12-16-13.gh-issue-75593.EFVhKR.rst deleted file mode 100644 index 9a31af9c110454..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-11-04-12-16-13.gh-issue-75593.EFVhKR.rst +++ /dev/null @@ -1 +0,0 @@ -Add support of :term:`path-like objects ` and :term:`bytes-like objects ` in :func:`wave.open`. diff --git a/Misc/NEWS.d/next/Library/2025-11-04-15-40-35.gh-issue-137969.9VZQVt.rst b/Misc/NEWS.d/next/Library/2025-11-04-15-40-35.gh-issue-137969.9VZQVt.rst deleted file mode 100644 index dfa582bdbc8825..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-11-04-15-40-35.gh-issue-137969.9VZQVt.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix :meth:`annotationlib.ForwardRef.evaluate` returning -:class:`~annotationlib.ForwardRef` objects which don't update with new -globals. diff --git a/Misc/NEWS.d/next/Library/2025-11-04-20-08-41.gh-issue-141018.d_oyOI.rst b/Misc/NEWS.d/next/Library/2025-11-04-20-08-41.gh-issue-141018.d_oyOI.rst deleted file mode 100644 index e776515a9fb267..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-11-04-20-08-41.gh-issue-141018.d_oyOI.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`mimetypes`: Update ``.exe``, ``.dll``, ``.rtf`` and (when -``strict=False``) ``.jpg`` to their correct IANA mime type. diff --git a/Misc/NEWS.d/next/Library/2025-11-06-15-11-50.gh-issue-141141.tgIfgH.rst b/Misc/NEWS.d/next/Library/2025-11-06-15-11-50.gh-issue-141141.tgIfgH.rst deleted file mode 100644 index f59ccfb33e7669..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-11-06-15-11-50.gh-issue-141141.tgIfgH.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a thread safety issue with :func:`base64.b85decode`. Contributed by Benel Tayar. diff --git a/Misc/NEWS.d/next/Library/2025-11-07-12-25-46.gh-issue-85524.9SWFIC.rst b/Misc/NEWS.d/next/Library/2025-11-07-12-25-46.gh-issue-85524.9SWFIC.rst deleted file mode 100644 index 3e4fd1a5897b04..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-11-07-12-25-46.gh-issue-85524.9SWFIC.rst +++ /dev/null @@ -1,3 +0,0 @@ -Update ``io.FileIO.readall``, an implementation of :meth:`io.RawIOBase.readall`, -to follow :class:`io.IOBase` guidelines and raise :exc:`io.UnsupportedOperation` -when a file is in "w" mode rather than :exc:`OSError` diff --git a/Misc/NEWS.d/next/Library/2025-11-08-13-03-10.gh-issue-87710.XJeZlP.rst b/Misc/NEWS.d/next/Library/2025-11-08-13-03-10.gh-issue-87710.XJeZlP.rst deleted file mode 100644 index 62073280e32b81..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-11-08-13-03-10.gh-issue-87710.XJeZlP.rst +++ /dev/null @@ -1 +0,0 @@ -:mod:`mimetypes`: Update mime type for ``.ai`` files to ``application/pdf``. diff --git a/Misc/NEWS.d/next/Library/2025-11-09-18-55-13.gh-issue-141311.qZ3swc.rst b/Misc/NEWS.d/next/Library/2025-11-09-18-55-13.gh-issue-141311.qZ3swc.rst deleted file mode 100644 index bb425ce5df309d..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-11-09-18-55-13.gh-issue-141311.qZ3swc.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix assertion failure in :func:`!io.BytesIO.readinto` and undefined behavior -arising when read position is above capcity in :class:`io.BytesIO`. diff --git a/Misc/NEWS.d/next/Library/2025-11-10-01-47-18.gh-issue-141314.baaa28.rst b/Misc/NEWS.d/next/Library/2025-11-10-01-47-18.gh-issue-141314.baaa28.rst deleted file mode 100644 index 37acaabfa3eada..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-11-10-01-47-18.gh-issue-141314.baaa28.rst +++ /dev/null @@ -1 +0,0 @@ -Fix assertion failure in :meth:`io.TextIOWrapper.tell` when reading files with standalone carriage return (``\r``) line endings. diff --git a/Misc/NEWS.d/next/Library/2025-11-12-01-49-03.gh-issue-137109.D6sq2B.rst b/Misc/NEWS.d/next/Library/2025-11-12-01-49-03.gh-issue-137109.D6sq2B.rst deleted file mode 100644 index 32f4e39f6d5f4c..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-11-12-01-49-03.gh-issue-137109.D6sq2B.rst +++ /dev/null @@ -1,5 +0,0 @@ -The :mod:`os.fork` and related forking APIs will no longer warn in the -common case where Linux or macOS platform APIs return the number of threads -in a process and find the answer to be 1 even when a -:func:`os.register_at_fork` ``after_in_parent=`` callback (re)starts a -thread. diff --git a/Misc/NEWS.d/next/Library/2025-11-12-15-42-47.gh-issue-124111.hTw4OE.rst b/Misc/NEWS.d/next/Library/2025-11-12-15-42-47.gh-issue-124111.hTw4OE.rst deleted file mode 100644 index 8436cd2415dbd6..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-11-12-15-42-47.gh-issue-124111.hTw4OE.rst +++ /dev/null @@ -1,2 +0,0 @@ -Updated Tcl threading configuration in :mod:`_tkinter` to assume that -threads are always available in Tcl 9 and later. diff --git a/Misc/NEWS.d/next/Library/2025-11-13-14-51-30.gh-issue-140938.kXsHHv.rst b/Misc/NEWS.d/next/Library/2025-11-13-14-51-30.gh-issue-140938.kXsHHv.rst deleted file mode 100644 index bd3044002a2d54..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-11-13-14-51-30.gh-issue-140938.kXsHHv.rst +++ /dev/null @@ -1,2 +0,0 @@ -The :func:`statistics.stdev` and :func:`statistics.pstdev` functions now raise a -:exc:`ValueError` when the input contains an infinity or a NaN. diff --git a/Misc/NEWS.d/next/Library/2025-11-14-16-24-20.gh-issue-141497.L_CxDJ.rst b/Misc/NEWS.d/next/Library/2025-11-14-16-24-20.gh-issue-141497.L_CxDJ.rst deleted file mode 100644 index 328bfe067ad96b..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-11-14-16-24-20.gh-issue-141497.L_CxDJ.rst +++ /dev/null @@ -1,4 +0,0 @@ -:mod:`ipaddress`: ensure that the methods -:meth:`IPv4Network.hosts() ` and -:meth:`IPv6Network.hosts() ` always return an -iterator. diff --git a/Misc/NEWS.d/next/Security/2025-05-30-22-33-27.gh-issue-136065.bu337o.rst b/Misc/NEWS.d/next/Security/2025-05-30-22-33-27.gh-issue-136065.bu337o.rst deleted file mode 100644 index 1d152bb5318380..00000000000000 --- a/Misc/NEWS.d/next/Security/2025-05-30-22-33-27.gh-issue-136065.bu337o.rst +++ /dev/null @@ -1 +0,0 @@ -Fix quadratic complexity in :func:`os.path.expandvars`. diff --git a/Misc/NEWS.d/next/Security/2025-06-28-13-23-53.gh-issue-136063.aGk0Jv.rst b/Misc/NEWS.d/next/Security/2025-06-28-13-23-53.gh-issue-136063.aGk0Jv.rst deleted file mode 100644 index 940a3ad5a72f68..00000000000000 --- a/Misc/NEWS.d/next/Security/2025-06-28-13-23-53.gh-issue-136063.aGk0Jv.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`email.message`: ensure linear complexity for legacy HTTP parameters -parsing. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Security/2025-08-15-23-08-44.gh-issue-137836.b55rhh.rst b/Misc/NEWS.d/next/Security/2025-08-15-23-08-44.gh-issue-137836.b55rhh.rst deleted file mode 100644 index c30c9439a76a19..00000000000000 --- a/Misc/NEWS.d/next/Security/2025-08-15-23-08-44.gh-issue-137836.b55rhh.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add support of the "plaintext" element, RAWTEXT elements "xmp", "iframe", -"noembed" and "noframes", and optionally RAWTEXT element "noscript" in -:class:`html.parser.HTMLParser`. diff --git a/Misc/NEWS.d/next/Tests/2025-07-09-21-45-51.gh-issue-136442.jlbklP.rst b/Misc/NEWS.d/next/Tests/2025-07-09-21-45-51.gh-issue-136442.jlbklP.rst deleted file mode 100644 index f87fb1113cad12..00000000000000 --- a/Misc/NEWS.d/next/Tests/2025-07-09-21-45-51.gh-issue-136442.jlbklP.rst +++ /dev/null @@ -1 +0,0 @@ -Use exitcode ``1`` instead of ``5`` if :func:`unittest.TestCase.setUpClass` raises an exception diff --git a/Misc/NEWS.d/next/Tests/2025-10-15-00-52-12.gh-issue-140082.fpET50.rst b/Misc/NEWS.d/next/Tests/2025-10-15-00-52-12.gh-issue-140082.fpET50.rst deleted file mode 100644 index 70e70218254488..00000000000000 --- a/Misc/NEWS.d/next/Tests/2025-10-15-00-52-12.gh-issue-140082.fpET50.rst +++ /dev/null @@ -1,3 +0,0 @@ -Update ``python -m test`` to set ``FORCE_COLOR=1`` when being run with color -enabled so that :mod:`unittest` which is run by it with redirected output will -output in color. diff --git a/Misc/NEWS.d/next/Tests/2025-10-23-16-39-49.gh-issue-140482.ZMtyeD.rst b/Misc/NEWS.d/next/Tests/2025-10-23-16-39-49.gh-issue-140482.ZMtyeD.rst deleted file mode 100644 index 20747ad7f113ec..00000000000000 --- a/Misc/NEWS.d/next/Tests/2025-10-23-16-39-49.gh-issue-140482.ZMtyeD.rst +++ /dev/null @@ -1 +0,0 @@ -Preserve and restore the state of ``stty echo`` as part of the test environment. diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-09-20-20-31-54.gh-issue-139188.zfcxkW.rst b/Misc/NEWS.d/next/Tools-Demos/2025-09-20-20-31-54.gh-issue-139188.zfcxkW.rst deleted file mode 100644 index 9f52d0163ab038..00000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2025-09-20-20-31-54.gh-issue-139188.zfcxkW.rst +++ /dev/null @@ -1 +0,0 @@ -Remove ``Tools/tz/zdump.py`` script. diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-09-21-10-30-08.gh-issue-139198.Fm7NfU.rst b/Misc/NEWS.d/next/Tools-Demos/2025-09-21-10-30-08.gh-issue-139198.Fm7NfU.rst deleted file mode 100644 index 0dc589c3986ad6..00000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2025-09-21-10-30-08.gh-issue-139198.Fm7NfU.rst +++ /dev/null @@ -1 +0,0 @@ -Remove ``Tools/scripts/checkpip.py`` script. diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-10-29-15-20-19.gh-issue-140702.ZXtW8h.rst b/Misc/NEWS.d/next/Tools-Demos/2025-10-29-15-20-19.gh-issue-140702.ZXtW8h.rst deleted file mode 100644 index 9efbf0162dd1c1..00000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2025-10-29-15-20-19.gh-issue-140702.ZXtW8h.rst +++ /dev/null @@ -1,2 +0,0 @@ -The iOS testbed app will now expose the ``GITHUB_ACTIONS`` environment -variable to iOS apps being tested. diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-11-12-12-54-28.gh-issue-141442.50dS3P.rst b/Misc/NEWS.d/next/Tools-Demos/2025-11-12-12-54-28.gh-issue-141442.50dS3P.rst deleted file mode 100644 index 073c070413f7e0..00000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2025-11-12-12-54-28.gh-issue-141442.50dS3P.rst +++ /dev/null @@ -1 +0,0 @@ -The iOS testbed now correctly handles test arguments that contain spaces. diff --git a/Misc/NEWS.d/next/Windows/2025-11-04-19-20-05.gh-issue-140849.YjB2ZZ.rst b/Misc/NEWS.d/next/Windows/2025-11-04-19-20-05.gh-issue-140849.YjB2ZZ.rst deleted file mode 100644 index 6f25b8675663fd..00000000000000 --- a/Misc/NEWS.d/next/Windows/2025-11-04-19-20-05.gh-issue-140849.YjB2ZZ.rst +++ /dev/null @@ -1 +0,0 @@ -Update bundled liblzma to version 5.8.1. diff --git a/README.rst b/README.rst index a228aafb09c704..bc1c1df2069558 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.15.0 alpha 1 +This is Python version 3.15.0 alpha 2 ===================================== .. image:: https://github.com/python/cpython/actions/workflows/build.yml/badge.svg?branch=main&event=push From a52c39e2608557a710784d5876150578d2ae5183 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Tue, 18 Nov 2025 15:14:16 +0000 Subject: [PATCH 091/638] gh-135953: Refactor test_sampling_profiler into multiple files (#141689) --- .../test_profiling/test_sampling_profiler.py | 3360 ----------------- .../test_sampling_profiler/__init__.py | 9 + .../test_sampling_profiler/helpers.py | 101 + .../test_sampling_profiler/mocks.py | 38 + .../test_sampling_profiler/test_advanced.py | 264 ++ .../test_sampling_profiler/test_cli.py | 664 ++++ .../test_sampling_profiler/test_collectors.py | 896 +++++ .../test_integration.py | 804 ++++ .../test_sampling_profiler/test_modes.py | 514 +++ .../test_sampling_profiler/test_profiler.py | 656 ++++ Makefile.pre.in | 1 + 11 files changed, 3947 insertions(+), 3360 deletions(-) delete mode 100644 Lib/test/test_profiling/test_sampling_profiler.py create mode 100644 Lib/test/test_profiling/test_sampling_profiler/__init__.py create mode 100644 Lib/test/test_profiling/test_sampling_profiler/helpers.py create mode 100644 Lib/test/test_profiling/test_sampling_profiler/mocks.py create mode 100644 Lib/test/test_profiling/test_sampling_profiler/test_advanced.py create mode 100644 Lib/test/test_profiling/test_sampling_profiler/test_cli.py create mode 100644 Lib/test/test_profiling/test_sampling_profiler/test_collectors.py create mode 100644 Lib/test/test_profiling/test_sampling_profiler/test_integration.py create mode 100644 Lib/test/test_profiling/test_sampling_profiler/test_modes.py create mode 100644 Lib/test/test_profiling/test_sampling_profiler/test_profiler.py diff --git a/Lib/test/test_profiling/test_sampling_profiler.py b/Lib/test/test_profiling/test_sampling_profiler.py deleted file mode 100644 index c2cc2ddd48a02c..00000000000000 --- a/Lib/test/test_profiling/test_sampling_profiler.py +++ /dev/null @@ -1,3360 +0,0 @@ -"""Tests for the sampling profiler (profiling.sampling).""" - -import contextlib -import io -import json -import marshal -import os -import shutil -import socket -import subprocess -import sys -import tempfile -import unittest -from collections import namedtuple -from unittest import mock - -from profiling.sampling.pstats_collector import PstatsCollector -from profiling.sampling.stack_collector import ( - CollapsedStackCollector, - FlamegraphCollector, -) -from profiling.sampling.gecko_collector import GeckoCollector - -from test.support.os_helper import unlink -from test.support import ( - force_not_colorized_test_class, - SHORT_TIMEOUT, - script_helper, - os_helper, - SuppressCrashReport, -) -from test.support.socket_helper import find_unused_port -from test.support import requires_subprocess, is_emscripten -from test.support import captured_stdout, captured_stderr - -PROCESS_VM_READV_SUPPORTED = False - -try: - from _remote_debugging import PROCESS_VM_READV_SUPPORTED - import _remote_debugging -except ImportError: - raise unittest.SkipTest( - "Test only runs when _remote_debugging is available" - ) -else: - import profiling.sampling - from profiling.sampling.sample import SampleProfiler - - - -class MockFrameInfo: - """Mock FrameInfo for testing since the real one isn't accessible.""" - - def __init__(self, filename, lineno, funcname): - self.filename = filename - self.lineno = lineno - self.funcname = funcname - - def __repr__(self): - return f"MockFrameInfo(filename='{self.filename}', lineno={self.lineno}, funcname='{self.funcname}')" - - -class MockThreadInfo: - """Mock ThreadInfo for testing since the real one isn't accessible.""" - - def __init__(self, thread_id, frame_info, status=0): # Default to THREAD_STATE_RUNNING (0) - self.thread_id = thread_id - self.frame_info = frame_info - self.status = status - - def __repr__(self): - return f"MockThreadInfo(thread_id={self.thread_id}, frame_info={self.frame_info}, status={self.status})" - - -class MockInterpreterInfo: - """Mock InterpreterInfo for testing since the real one isn't accessible.""" - - def __init__(self, interpreter_id, threads): - self.interpreter_id = interpreter_id - self.threads = threads - - def __repr__(self): - return f"MockInterpreterInfo(interpreter_id={self.interpreter_id}, threads={self.threads})" - - -skip_if_not_supported = unittest.skipIf( - ( - sys.platform != "darwin" - and sys.platform != "linux" - and sys.platform != "win32" - ), - "Test only runs on Linux, Windows and MacOS", -) - -SubprocessInfo = namedtuple('SubprocessInfo', ['process', 'socket']) - - -@contextlib.contextmanager -def test_subprocess(script): - # Find an unused port for socket communication - port = find_unused_port() - - # Inject socket connection code at the beginning of the script - socket_code = f''' -import socket -_test_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) -_test_sock.connect(('localhost', {port})) -_test_sock.sendall(b"ready") -''' - - # Combine socket code with user script - full_script = socket_code + script - - # Create server socket to wait for process to be ready - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server_socket.bind(("localhost", port)) - server_socket.settimeout(SHORT_TIMEOUT) - server_socket.listen(1) - - proc = subprocess.Popen( - [sys.executable, "-c", full_script], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - - client_socket = None - try: - # Wait for process to connect and send ready signal - client_socket, _ = server_socket.accept() - server_socket.close() - response = client_socket.recv(1024) - if response != b"ready": - raise RuntimeError(f"Unexpected response from subprocess: {response}") - - yield SubprocessInfo(proc, client_socket) - finally: - if client_socket is not None: - client_socket.close() - if proc.poll() is None: - proc.kill() - proc.wait() - - -def close_and_unlink(file): - file.close() - unlink(file.name) - - -class TestSampleProfilerComponents(unittest.TestCase): - """Unit tests for individual profiler components.""" - - def test_mock_frame_info_with_empty_and_unicode_values(self): - """Test MockFrameInfo handles empty strings, unicode characters, and very long names correctly.""" - # Test with empty strings - frame = MockFrameInfo("", 0, "") - self.assertEqual(frame.filename, "") - self.assertEqual(frame.lineno, 0) - self.assertEqual(frame.funcname, "") - self.assertIn("filename=''", repr(frame)) - - # Test with unicode characters - frame = MockFrameInfo("文件.py", 42, "函数名") - self.assertEqual(frame.filename, "文件.py") - self.assertEqual(frame.funcname, "函数名") - - # Test with very long names - long_filename = "x" * 1000 + ".py" - long_funcname = "func_" + "x" * 1000 - frame = MockFrameInfo(long_filename, 999999, long_funcname) - self.assertEqual(frame.filename, long_filename) - self.assertEqual(frame.lineno, 999999) - self.assertEqual(frame.funcname, long_funcname) - - def test_pstats_collector_with_extreme_intervals_and_empty_data(self): - """Test PstatsCollector handles zero/large intervals, empty frames, None thread IDs, and duplicate frames.""" - # Test with zero interval - collector = PstatsCollector(sample_interval_usec=0) - self.assertEqual(collector.sample_interval_usec, 0) - - # Test with very large interval - collector = PstatsCollector(sample_interval_usec=1000000000) - self.assertEqual(collector.sample_interval_usec, 1000000000) - - # Test collecting empty frames list - collector = PstatsCollector(sample_interval_usec=1000) - collector.collect([]) - self.assertEqual(len(collector.result), 0) - - # Test collecting frames with None thread id - test_frames = [MockInterpreterInfo(0, [MockThreadInfo(None, [MockFrameInfo("file.py", 10, "func")])])] - collector.collect(test_frames) - # Should still process the frames - self.assertEqual(len(collector.result), 1) - - # Test collecting duplicate frames in same sample - test_frames = [ - MockInterpreterInfo( - 0, # interpreter_id - [MockThreadInfo( - 1, - [ - MockFrameInfo("file.py", 10, "func1"), - MockFrameInfo("file.py", 10, "func1"), # Duplicate - ], - )] - ) - ] - collector = PstatsCollector(sample_interval_usec=1000) - collector.collect(test_frames) - # Should count both occurrences - self.assertEqual( - collector.result[("file.py", 10, "func1")]["cumulative_calls"], 2 - ) - - def test_pstats_collector_single_frame_stacks(self): - """Test PstatsCollector with single-frame call stacks to trigger len(frames) <= 1 branch.""" - collector = PstatsCollector(sample_interval_usec=1000) - - # Test with exactly one frame (should trigger the <= 1 condition) - single_frame = [MockInterpreterInfo(0, [MockThreadInfo(1, [MockFrameInfo("single.py", 10, "single_func")])])] - collector.collect(single_frame) - - # Should record the single frame with inline call - self.assertEqual(len(collector.result), 1) - single_key = ("single.py", 10, "single_func") - self.assertIn(single_key, collector.result) - self.assertEqual(collector.result[single_key]["direct_calls"], 1) - self.assertEqual(collector.result[single_key]["cumulative_calls"], 1) - - # Test with empty frames (should also trigger <= 1 condition) - empty_frames = [MockInterpreterInfo(0, [MockThreadInfo(1, [])])] - collector.collect(empty_frames) - - # Should not add any new entries - self.assertEqual( - len(collector.result), 1 - ) # Still just the single frame - - # Test mixed single and multi-frame stacks - mixed_frames = [ - MockInterpreterInfo( - 0, - [ - MockThreadInfo( - 1, - [MockFrameInfo("single2.py", 20, "single_func2")], - ), # Single frame - MockThreadInfo( - 2, - [ # Multi-frame stack - MockFrameInfo("multi.py", 30, "multi_func1"), - MockFrameInfo("multi.py", 40, "multi_func2"), - ], - ), - ] - ), - ] - collector.collect(mixed_frames) - - # Should have recorded all functions - self.assertEqual( - len(collector.result), 4 - ) # single + single2 + multi1 + multi2 - - # Verify single frame handling - single2_key = ("single2.py", 20, "single_func2") - self.assertIn(single2_key, collector.result) - self.assertEqual(collector.result[single2_key]["direct_calls"], 1) - self.assertEqual(collector.result[single2_key]["cumulative_calls"], 1) - - # Verify multi-frame handling still works - multi1_key = ("multi.py", 30, "multi_func1") - multi2_key = ("multi.py", 40, "multi_func2") - self.assertIn(multi1_key, collector.result) - self.assertIn(multi2_key, collector.result) - self.assertEqual(collector.result[multi1_key]["direct_calls"], 1) - self.assertEqual( - collector.result[multi2_key]["cumulative_calls"], 1 - ) # Called from multi1 - - def test_collapsed_stack_collector_with_empty_and_deep_stacks(self): - """Test CollapsedStackCollector handles empty frames, single-frame stacks, and very deep call stacks.""" - collector = CollapsedStackCollector() - - # Test with empty frames - collector.collect([]) - self.assertEqual(len(collector.stack_counter), 0) - - # Test with single frame stack - test_frames = [MockInterpreterInfo(0, [MockThreadInfo(1, [("file.py", 10, "func")])])] - collector.collect(test_frames) - self.assertEqual(len(collector.stack_counter), 1) - ((path, thread_id), count), = collector.stack_counter.items() - self.assertEqual(path, (("file.py", 10, "func"),)) - self.assertEqual(thread_id, 1) - self.assertEqual(count, 1) - - # Test with very deep stack - deep_stack = [(f"file{i}.py", i, f"func{i}") for i in range(100)] - test_frames = [MockInterpreterInfo(0, [MockThreadInfo(1, deep_stack)])] - collector = CollapsedStackCollector() - collector.collect(test_frames) - # One aggregated path with 100 frames (reversed) - ((path_tuple, thread_id),), = (collector.stack_counter.keys(),) - self.assertEqual(len(path_tuple), 100) - self.assertEqual(path_tuple[0], ("file99.py", 99, "func99")) - self.assertEqual(path_tuple[-1], ("file0.py", 0, "func0")) - self.assertEqual(thread_id, 1) - - def test_pstats_collector_basic(self): - """Test basic PstatsCollector functionality.""" - collector = PstatsCollector(sample_interval_usec=1000) - - # Test empty state - self.assertEqual(len(collector.result), 0) - self.assertEqual(len(collector.stats), 0) - - # Test collecting sample data - test_frames = [ - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ - MockFrameInfo("file.py", 10, "func1"), - MockFrameInfo("file.py", 20, "func2"), - ], - )] - ) - ] - collector.collect(test_frames) - - # Should have recorded calls for both functions - self.assertEqual(len(collector.result), 2) - self.assertIn(("file.py", 10, "func1"), collector.result) - self.assertIn(("file.py", 20, "func2"), collector.result) - - # Top-level function should have direct call - self.assertEqual( - collector.result[("file.py", 10, "func1")]["direct_calls"], 1 - ) - self.assertEqual( - collector.result[("file.py", 10, "func1")]["cumulative_calls"], 1 - ) - - # Calling function should have cumulative call but no direct calls - self.assertEqual( - collector.result[("file.py", 20, "func2")]["cumulative_calls"], 1 - ) - self.assertEqual( - collector.result[("file.py", 20, "func2")]["direct_calls"], 0 - ) - - def test_pstats_collector_create_stats(self): - """Test PstatsCollector stats creation.""" - collector = PstatsCollector( - sample_interval_usec=1000000 - ) # 1 second intervals - - test_frames = [ - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ - MockFrameInfo("file.py", 10, "func1"), - MockFrameInfo("file.py", 20, "func2"), - ], - )] - ) - ] - collector.collect(test_frames) - collector.collect(test_frames) # Collect twice - - collector.create_stats() - - # Check stats format: (direct_calls, cumulative_calls, tt, ct, callers) - func1_stats = collector.stats[("file.py", 10, "func1")] - self.assertEqual(func1_stats[0], 2) # direct_calls (top of stack) - self.assertEqual(func1_stats[1], 2) # cumulative_calls - self.assertEqual( - func1_stats[2], 2.0 - ) # tt (total time - 2 samples * 1 sec) - self.assertEqual(func1_stats[3], 2.0) # ct (cumulative time) - - func2_stats = collector.stats[("file.py", 20, "func2")] - self.assertEqual( - func2_stats[0], 0 - ) # direct_calls (never top of stack) - self.assertEqual( - func2_stats[1], 2 - ) # cumulative_calls (appears in stack) - self.assertEqual(func2_stats[2], 0.0) # tt (no direct calls) - self.assertEqual(func2_stats[3], 2.0) # ct (cumulative time) - - def test_collapsed_stack_collector_basic(self): - collector = CollapsedStackCollector() - - # Test empty state - self.assertEqual(len(collector.stack_counter), 0) - - # Test collecting sample data - test_frames = [ - MockInterpreterInfo(0, [MockThreadInfo(1, [("file.py", 10, "func1"), ("file.py", 20, "func2")])]) - ] - collector.collect(test_frames) - - # Should store one reversed path - self.assertEqual(len(collector.stack_counter), 1) - ((path, thread_id), count), = collector.stack_counter.items() - expected_tree = (("file.py", 20, "func2"), ("file.py", 10, "func1")) - self.assertEqual(path, expected_tree) - self.assertEqual(thread_id, 1) - self.assertEqual(count, 1) - - def test_collapsed_stack_collector_export(self): - collapsed_out = tempfile.NamedTemporaryFile(delete=False) - self.addCleanup(close_and_unlink, collapsed_out) - - collector = CollapsedStackCollector() - - test_frames1 = [ - MockInterpreterInfo(0, [MockThreadInfo(1, [("file.py", 10, "func1"), ("file.py", 20, "func2")])]) - ] - test_frames2 = [ - MockInterpreterInfo(0, [MockThreadInfo(1, [("file.py", 10, "func1"), ("file.py", 20, "func2")])]) - ] # Same stack - test_frames3 = [MockInterpreterInfo(0, [MockThreadInfo(1, [("other.py", 5, "other_func")])])] - - collector.collect(test_frames1) - collector.collect(test_frames2) - collector.collect(test_frames3) - - with (captured_stdout(), captured_stderr()): - collector.export(collapsed_out.name) - # Check file contents - with open(collapsed_out.name, "r") as f: - content = f.read() - - lines = content.strip().split("\n") - self.assertEqual(len(lines), 2) # Two unique stacks - - # Check collapsed format: tid:X;file:func:line;file:func:line count - stack1_expected = "tid:1;file.py:func2:20;file.py:func1:10 2" - stack2_expected = "tid:1;other.py:other_func:5 1" - - self.assertIn(stack1_expected, lines) - self.assertIn(stack2_expected, lines) - - def test_flamegraph_collector_basic(self): - """Test basic FlamegraphCollector functionality.""" - collector = FlamegraphCollector() - - # Empty collector should produce 'No Data' - data = collector._convert_to_flamegraph_format() - # With string table, name is now an index - resolve it using the strings array - strings = data.get("strings", []) - name_index = data.get("name", 0) - resolved_name = strings[name_index] if isinstance(name_index, int) and 0 <= name_index < len(strings) else str(name_index) - self.assertIn(resolved_name, ("No Data", "No significant data")) - - # Test collecting sample data - test_frames = [ - MockInterpreterInfo( - 0, - [MockThreadInfo(1, [("file.py", 10, "func1"), ("file.py", 20, "func2")])], - ) - ] - collector.collect(test_frames) - - # Convert and verify structure: func2 -> func1 with counts = 1 - data = collector._convert_to_flamegraph_format() - # Expect promotion: root is the single child (func2), with func1 as its only child - strings = data.get("strings", []) - name_index = data.get("name", 0) - name = strings[name_index] if isinstance(name_index, int) and 0 <= name_index < len(strings) else str(name_index) - self.assertIsInstance(name, str) - self.assertTrue(name.startswith("Program Root: ")) - self.assertIn("func2 (file.py:20)", name) # formatted name - children = data.get("children", []) - self.assertEqual(len(children), 1) - child = children[0] - child_name_index = child.get("name", 0) - child_name = strings[child_name_index] if isinstance(child_name_index, int) and 0 <= child_name_index < len(strings) else str(child_name_index) - self.assertIn("func1 (file.py:10)", child_name) # formatted name - self.assertEqual(child["value"], 1) - - def test_flamegraph_collector_export(self): - """Test flamegraph HTML export functionality.""" - flamegraph_out = tempfile.NamedTemporaryFile( - suffix=".html", delete=False - ) - self.addCleanup(close_and_unlink, flamegraph_out) - - collector = FlamegraphCollector() - - # Create some test data (use Interpreter/Thread objects like runtime) - test_frames1 = [ - MockInterpreterInfo( - 0, - [MockThreadInfo(1, [("file.py", 10, "func1"), ("file.py", 20, "func2")])], - ) - ] - test_frames2 = [ - MockInterpreterInfo( - 0, - [MockThreadInfo(1, [("file.py", 10, "func1"), ("file.py", 20, "func2")])], - ) - ] # Same stack - test_frames3 = [ - MockInterpreterInfo(0, [MockThreadInfo(1, [("other.py", 5, "other_func")])]) - ] - - collector.collect(test_frames1) - collector.collect(test_frames2) - collector.collect(test_frames3) - - # Export flamegraph - with (captured_stdout(), captured_stderr()): - collector.export(flamegraph_out.name) - - # Verify file was created and contains valid data - self.assertTrue(os.path.exists(flamegraph_out.name)) - self.assertGreater(os.path.getsize(flamegraph_out.name), 0) - - # Check file contains HTML content - with open(flamegraph_out.name, "r", encoding="utf-8") as f: - content = f.read() - - # Should be valid HTML - self.assertIn("", content.lower()) - self.assertIn(" 0) - self.assertGreater(mock_collector.collect.call_count, 0) - self.assertLessEqual(mock_collector.collect.call_count, 3) - - def test_sample_profiler_missed_samples_warning(self): - """Test that the profiler warns about missed samples when sampling is too slow.""" - from profiling.sampling.sample import SampleProfiler - - mock_unwinder = mock.MagicMock() - mock_unwinder.get_stack_trace.return_value = [ - ( - 1, - [ - mock.MagicMock( - filename="test.py", lineno=10, funcname="test_func" - ) - ], - ) - ] - - with mock.patch( - "_remote_debugging.RemoteUnwinder" - ) as mock_unwinder_class: - mock_unwinder_class.return_value = mock_unwinder - - # Use very short interval that we'll miss - profiler = SampleProfiler( - pid=12345, sample_interval_usec=1000, all_threads=False - ) # 1ms interval - - mock_collector = mock.MagicMock() - - # Simulate slow sampling where we miss many samples - times = [ - 0.0, - 0.1, - 0.2, - 0.3, - 0.4, - 0.5, - 0.6, - 0.7, - ] # Extra time points to avoid StopIteration - - with mock.patch("time.perf_counter", side_effect=times): - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - profiler.sample(mock_collector, duration_sec=0.5) - - result = output.getvalue() - - # Should warn about missed samples - self.assertIn("Warning: missed", result) - self.assertIn("samples from the expected total", result) - - -@force_not_colorized_test_class -class TestPrintSampledStats(unittest.TestCase): - """Test the print_sampled_stats function.""" - - def setUp(self): - """Set up test data.""" - # Mock stats data - self.mock_stats = mock.MagicMock() - self.mock_stats.stats = { - ("file1.py", 10, "func1"): ( - 100, - 100, - 0.5, - 0.5, - {}, - ), # cc, nc, tt, ct, callers - ("file2.py", 20, "func2"): (50, 50, 0.25, 0.3, {}), - ("file3.py", 30, "func3"): (200, 200, 1.5, 2.0, {}), - ("file4.py", 40, "func4"): ( - 10, - 10, - 0.001, - 0.001, - {}, - ), # millisecond range - ("file5.py", 50, "func5"): ( - 5, - 5, - 0.000001, - 0.000002, - {}, - ), # microsecond range - } - - def test_print_sampled_stats_basic(self): - """Test basic print_sampled_stats functionality.""" - from profiling.sampling.sample import print_sampled_stats - - # Capture output - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats(self.mock_stats, sample_interval_usec=100) - - result = output.getvalue() - - # Check header is present - self.assertIn("Profile Stats:", result) - self.assertIn("nsamples", result) - self.assertIn("tottime", result) - self.assertIn("cumtime", result) - - # Check functions are present - self.assertIn("func1", result) - self.assertIn("func2", result) - self.assertIn("func3", result) - - def test_print_sampled_stats_sorting(self): - """Test different sorting options.""" - from profiling.sampling.sample import print_sampled_stats - - # Test sort by calls - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats( - self.mock_stats, sort=0, sample_interval_usec=100 - ) - - result = output.getvalue() - lines = result.strip().split("\n") - - # Find the data lines (skip header) - data_lines = [l for l in lines if "file" in l and ".py" in l] - # func3 should be first (200 calls) - self.assertIn("func3", data_lines[0]) - - # Test sort by time - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats( - self.mock_stats, sort=1, sample_interval_usec=100 - ) - - result = output.getvalue() - lines = result.strip().split("\n") - - data_lines = [l for l in lines if "file" in l and ".py" in l] - # func3 should be first (1.5s time) - self.assertIn("func3", data_lines[0]) - - def test_print_sampled_stats_limit(self): - """Test limiting output rows.""" - from profiling.sampling.sample import print_sampled_stats - - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats( - self.mock_stats, limit=2, sample_interval_usec=100 - ) - - result = output.getvalue() - - # Count function entries in the main stats section (not in summary) - lines = result.split("\n") - # Find where the main stats section ends (before summary) - main_section_lines = [] - for line in lines: - if "Summary of Interesting Functions:" in line: - break - main_section_lines.append(line) - - # Count function entries only in main section - func_count = sum( - 1 - for line in main_section_lines - if "func" in line and ".py" in line - ) - self.assertEqual(func_count, 2) - - def test_print_sampled_stats_time_units(self): - """Test proper time unit selection.""" - from profiling.sampling.sample import print_sampled_stats - - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats(self.mock_stats, sample_interval_usec=100) - - result = output.getvalue() - - # Should use seconds for the header since max time is > 1s - self.assertIn("tottime (s)", result) - self.assertIn("cumtime (s)", result) - - # Test with only microsecond-range times - micro_stats = mock.MagicMock() - micro_stats.stats = { - ("file1.py", 10, "func1"): (100, 100, 0.000005, 0.000010, {}), - } - - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats(micro_stats, sample_interval_usec=100) - - result = output.getvalue() - - # Should use microseconds - self.assertIn("tottime (μs)", result) - self.assertIn("cumtime (μs)", result) - - def test_print_sampled_stats_summary(self): - """Test summary section generation.""" - from profiling.sampling.sample import print_sampled_stats - - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats( - self.mock_stats, - show_summary=True, - sample_interval_usec=100, - ) - - result = output.getvalue() - - # Check summary sections are present - self.assertIn("Summary of Interesting Functions:", result) - self.assertIn( - "Functions with Highest Direct/Cumulative Ratio (Hot Spots):", - result, - ) - self.assertIn( - "Functions with Highest Call Frequency (Indirect Calls):", result - ) - self.assertIn( - "Functions with Highest Call Magnification (Cumulative/Direct):", - result, - ) - - def test_print_sampled_stats_no_summary(self): - """Test disabling summary output.""" - from profiling.sampling.sample import print_sampled_stats - - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats( - self.mock_stats, - show_summary=False, - sample_interval_usec=100, - ) - - result = output.getvalue() - - # Summary should not be present - self.assertNotIn("Summary of Interesting Functions:", result) - - def test_print_sampled_stats_empty_stats(self): - """Test with empty stats.""" - from profiling.sampling.sample import print_sampled_stats - - empty_stats = mock.MagicMock() - empty_stats.stats = {} - - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats(empty_stats, sample_interval_usec=100) - - result = output.getvalue() - - # Should still print header - self.assertIn("Profile Stats:", result) - - def test_print_sampled_stats_sample_percentage_sorting(self): - """Test sample percentage sorting options.""" - from profiling.sampling.sample import print_sampled_stats - - # Add a function with high sample percentage (more direct calls than func3's 200) - self.mock_stats.stats[("expensive.py", 60, "expensive_func")] = ( - 300, # direct calls (higher than func3's 200) - 300, # cumulative calls - 1.0, # total time - 1.0, # cumulative time - {}, - ) - - # Test sort by sample percentage - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats( - self.mock_stats, sort=3, sample_interval_usec=100 - ) # sample percentage - - result = output.getvalue() - lines = result.strip().split("\n") - - data_lines = [l for l in lines if ".py" in l and "func" in l] - # expensive_func should be first (highest sample percentage) - self.assertIn("expensive_func", data_lines[0]) - - def test_print_sampled_stats_with_recursive_calls(self): - """Test print_sampled_stats with recursive calls where nc != cc.""" - from profiling.sampling.sample import print_sampled_stats - - # Create stats with recursive calls (nc != cc) - recursive_stats = mock.MagicMock() - recursive_stats.stats = { - # (direct_calls, cumulative_calls, tt, ct, callers) - recursive function - ("recursive.py", 10, "factorial"): ( - 5, # direct_calls - 10, # cumulative_calls (appears more times in stack due to recursion) - 0.5, - 0.6, - {}, - ), - ("normal.py", 20, "normal_func"): ( - 3, # direct_calls - 3, # cumulative_calls (same as direct for non-recursive) - 0.2, - 0.2, - {}, - ), - } - - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats(recursive_stats, sample_interval_usec=100) - - result = output.getvalue() - - # Should display recursive calls as "5/10" format - self.assertIn("5/10", result) # nc/cc format for recursive calls - self.assertIn("3", result) # just nc for non-recursive calls - self.assertIn("factorial", result) - self.assertIn("normal_func", result) - - def test_print_sampled_stats_with_zero_call_counts(self): - """Test print_sampled_stats with zero call counts to trigger division protection.""" - from profiling.sampling.sample import print_sampled_stats - - # Create stats with zero call counts - zero_stats = mock.MagicMock() - zero_stats.stats = { - ("file.py", 10, "zero_calls"): (0, 0, 0.0, 0.0, {}), # Zero calls - ("file.py", 20, "normal_func"): ( - 5, - 5, - 0.1, - 0.1, - {}, - ), # Normal function - } - - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats(zero_stats, sample_interval_usec=100) - - result = output.getvalue() - - # Should handle zero call counts gracefully - self.assertIn("zero_calls", result) - self.assertIn("zero_calls", result) - self.assertIn("normal_func", result) - - def test_print_sampled_stats_sort_by_name(self): - """Test sort by function name option.""" - from profiling.sampling.sample import print_sampled_stats - - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats( - self.mock_stats, sort=-1, sample_interval_usec=100 - ) # sort by name - - result = output.getvalue() - lines = result.strip().split("\n") - - # Find the data lines (skip header and summary) - # Data lines start with whitespace and numbers, and contain filename:lineno(function) - data_lines = [] - for line in lines: - # Skip header lines and summary sections - if ( - line.startswith(" ") - and "(" in line - and ")" in line - and not line.startswith( - " 1." - ) # Skip summary lines that start with times - and not line.startswith( - " 0." - ) # Skip summary lines that start with times - and not "per call" in line # Skip summary lines - and not "calls" in line # Skip summary lines - and not "total time" in line # Skip summary lines - and not "cumulative time" in line - ): # Skip summary lines - data_lines.append(line) - - # Extract just the function names for comparison - func_names = [] - import re - - for line in data_lines: - # Function name is between the last ( and ), accounting for ANSI color codes - match = re.search(r"\(([^)]+)\)$", line) - if match: - func_name = match.group(1) - # Remove ANSI color codes - func_name = re.sub(r"\x1b\[[0-9;]*m", "", func_name) - func_names.append(func_name) - - # Verify we extracted function names and they are sorted - self.assertGreater( - len(func_names), 0, "Should have extracted some function names" - ) - self.assertEqual( - func_names, - sorted(func_names), - f"Function names {func_names} should be sorted alphabetically", - ) - - def test_print_sampled_stats_with_zero_time_functions(self): - """Test summary sections with functions that have zero time.""" - from profiling.sampling.sample import print_sampled_stats - - # Create stats with zero-time functions - zero_time_stats = mock.MagicMock() - zero_time_stats.stats = { - ("file1.py", 10, "zero_time_func"): ( - 5, - 5, - 0.0, - 0.0, - {}, - ), # Zero time - ("file2.py", 20, "normal_func"): ( - 3, - 3, - 0.1, - 0.1, - {}, - ), # Normal time - } - - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats( - zero_time_stats, - show_summary=True, - sample_interval_usec=100, - ) - - result = output.getvalue() - - # Should handle zero-time functions gracefully in summary - self.assertIn("Summary of Interesting Functions:", result) - self.assertIn("zero_time_func", result) - self.assertIn("normal_func", result) - - def test_print_sampled_stats_with_malformed_qualified_names(self): - """Test summary generation with function names that don't contain colons.""" - from profiling.sampling.sample import print_sampled_stats - - # Create stats with function names that would create malformed qualified names - malformed_stats = mock.MagicMock() - malformed_stats.stats = { - # Function name without clear module separation - ("no_colon_func", 10, "func"): (3, 3, 0.1, 0.1, {}), - ("", 20, "empty_filename_func"): (2, 2, 0.05, 0.05, {}), - ("normal.py", 30, "normal_func"): (5, 5, 0.2, 0.2, {}), - } - - with io.StringIO() as output: - with mock.patch("sys.stdout", output): - print_sampled_stats( - malformed_stats, - show_summary=True, - sample_interval_usec=100, - ) - - result = output.getvalue() - - # Should handle malformed names gracefully in summary aggregation - self.assertIn("Summary of Interesting Functions:", result) - # All function names should appear somewhere in the output - self.assertIn("func", result) - self.assertIn("empty_filename_func", result) - self.assertIn("normal_func", result) - - def test_print_sampled_stats_with_recursive_call_stats_creation(self): - """Test create_stats with recursive call data to trigger total_rec_calls branch.""" - collector = PstatsCollector(sample_interval_usec=1000000) # 1 second - - # Simulate recursive function data where total_rec_calls would be set - # We need to manually manipulate the collector result to test this branch - collector.result = { - ("recursive.py", 10, "factorial"): { - "total_rec_calls": 3, # Non-zero recursive calls - "direct_calls": 5, - "cumulative_calls": 10, - }, - ("normal.py", 20, "normal_func"): { - "total_rec_calls": 0, # Zero recursive calls - "direct_calls": 2, - "cumulative_calls": 5, - }, - } - - collector.create_stats() - - # Check that recursive calls are handled differently from non-recursive - factorial_stats = collector.stats[("recursive.py", 10, "factorial")] - normal_stats = collector.stats[("normal.py", 20, "normal_func")] - - # factorial should use cumulative_calls (10) as nc - self.assertEqual( - factorial_stats[1], 10 - ) # nc should be cumulative_calls - self.assertEqual(factorial_stats[0], 5) # cc should be direct_calls - - # normal_func should use cumulative_calls as nc - self.assertEqual(normal_stats[1], 5) # nc should be cumulative_calls - self.assertEqual(normal_stats[0], 2) # cc should be direct_calls - - -@skip_if_not_supported -@unittest.skipIf( - sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, - "Test only runs on Linux with process_vm_readv support", -) -class TestRecursiveFunctionProfiling(unittest.TestCase): - """Test profiling of recursive functions and complex call patterns.""" - - def test_recursive_function_call_counting(self): - """Test that recursive function calls are counted correctly.""" - collector = PstatsCollector(sample_interval_usec=1000) - - # Simulate a recursive call pattern: fibonacci(5) calling itself - recursive_frames = [ - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ # First sample: deep in recursion - MockFrameInfo("fib.py", 10, "fibonacci"), - MockFrameInfo("fib.py", 10, "fibonacci"), # recursive call - MockFrameInfo( - "fib.py", 10, "fibonacci" - ), # deeper recursion - MockFrameInfo("fib.py", 10, "fibonacci"), # even deeper - MockFrameInfo("main.py", 5, "main"), # main caller - ], - )] - ), - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ # Second sample: different recursion depth - MockFrameInfo("fib.py", 10, "fibonacci"), - MockFrameInfo("fib.py", 10, "fibonacci"), # recursive call - MockFrameInfo("main.py", 5, "main"), # main caller - ], - )] - ), - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ # Third sample: back to deeper recursion - MockFrameInfo("fib.py", 10, "fibonacci"), - MockFrameInfo("fib.py", 10, "fibonacci"), - MockFrameInfo("fib.py", 10, "fibonacci"), - MockFrameInfo("main.py", 5, "main"), - ], - )] - ), - ] - - for frames in recursive_frames: - collector.collect([frames]) - - collector.create_stats() - - # Check that recursive calls are counted properly - fib_key = ("fib.py", 10, "fibonacci") - main_key = ("main.py", 5, "main") - - self.assertIn(fib_key, collector.stats) - self.assertIn(main_key, collector.stats) - - # Fibonacci should have many calls due to recursion - fib_stats = collector.stats[fib_key] - direct_calls, cumulative_calls, tt, ct, callers = fib_stats - - # Should have recorded multiple calls (9 total appearances in samples) - self.assertEqual(cumulative_calls, 9) - self.assertGreater(tt, 0) # Should have some total time - self.assertGreater(ct, 0) # Should have some cumulative time - - # Main should have fewer calls - main_stats = collector.stats[main_key] - main_direct_calls, main_cumulative_calls = main_stats[0], main_stats[1] - self.assertEqual(main_direct_calls, 0) # Never directly executing - self.assertEqual(main_cumulative_calls, 3) # Appears in all 3 samples - - def test_nested_function_hierarchy(self): - """Test profiling of deeply nested function calls.""" - collector = PstatsCollector(sample_interval_usec=1000) - - # Simulate a deep call hierarchy - deep_call_frames = [ - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ - MockFrameInfo("level1.py", 10, "level1_func"), - MockFrameInfo("level2.py", 20, "level2_func"), - MockFrameInfo("level3.py", 30, "level3_func"), - MockFrameInfo("level4.py", 40, "level4_func"), - MockFrameInfo("level5.py", 50, "level5_func"), - MockFrameInfo("main.py", 5, "main"), - ], - )] - ), - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ # Same hierarchy sampled again - MockFrameInfo("level1.py", 10, "level1_func"), - MockFrameInfo("level2.py", 20, "level2_func"), - MockFrameInfo("level3.py", 30, "level3_func"), - MockFrameInfo("level4.py", 40, "level4_func"), - MockFrameInfo("level5.py", 50, "level5_func"), - MockFrameInfo("main.py", 5, "main"), - ], - )] - ), - ] - - for frames in deep_call_frames: - collector.collect([frames]) - - collector.create_stats() - - # All levels should be recorded - for level in range(1, 6): - key = (f"level{level}.py", level * 10, f"level{level}_func") - self.assertIn(key, collector.stats) - - stats = collector.stats[key] - direct_calls, cumulative_calls, tt, ct, callers = stats - - # Each level should appear in stack twice (2 samples) - self.assertEqual(cumulative_calls, 2) - - # Only level1 (deepest) should have direct calls - if level == 1: - self.assertEqual(direct_calls, 2) - else: - self.assertEqual(direct_calls, 0) - - # Deeper levels should have lower cumulative time than higher levels - # (since they don't include time from functions they call) - if level == 1: # Deepest level with most time - self.assertGreater(ct, 0) - - def test_alternating_call_patterns(self): - """Test profiling with alternating call patterns.""" - collector = PstatsCollector(sample_interval_usec=1000) - - # Simulate alternating execution paths - pattern_frames = [ - # Pattern A: path through func_a - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ - MockFrameInfo("module.py", 10, "func_a"), - MockFrameInfo("module.py", 30, "shared_func"), - MockFrameInfo("main.py", 5, "main"), - ], - )] - ), - # Pattern B: path through func_b - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ - MockFrameInfo("module.py", 20, "func_b"), - MockFrameInfo("module.py", 30, "shared_func"), - MockFrameInfo("main.py", 5, "main"), - ], - )] - ), - # Pattern A again - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ - MockFrameInfo("module.py", 10, "func_a"), - MockFrameInfo("module.py", 30, "shared_func"), - MockFrameInfo("main.py", 5, "main"), - ], - )] - ), - # Pattern B again - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ - MockFrameInfo("module.py", 20, "func_b"), - MockFrameInfo("module.py", 30, "shared_func"), - MockFrameInfo("main.py", 5, "main"), - ], - )] - ), - ] - - for frames in pattern_frames: - collector.collect([frames]) - - collector.create_stats() - - # Check that both paths are recorded equally - func_a_key = ("module.py", 10, "func_a") - func_b_key = ("module.py", 20, "func_b") - shared_key = ("module.py", 30, "shared_func") - main_key = ("main.py", 5, "main") - - # func_a and func_b should each be directly executing twice - self.assertEqual(collector.stats[func_a_key][0], 2) # direct_calls - self.assertEqual(collector.stats[func_a_key][1], 2) # cumulative_calls - self.assertEqual(collector.stats[func_b_key][0], 2) # direct_calls - self.assertEqual(collector.stats[func_b_key][1], 2) # cumulative_calls - - # shared_func should appear in all samples (4 times) but never directly executing - self.assertEqual(collector.stats[shared_key][0], 0) # direct_calls - self.assertEqual(collector.stats[shared_key][1], 4) # cumulative_calls - - # main should appear in all samples but never directly executing - self.assertEqual(collector.stats[main_key][0], 0) # direct_calls - self.assertEqual(collector.stats[main_key][1], 4) # cumulative_calls - - def test_collapsed_stack_with_recursion(self): - """Test collapsed stack collector with recursive patterns.""" - collector = CollapsedStackCollector() - - # Recursive call pattern - recursive_frames = [ - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ - ("factorial.py", 10, "factorial"), - ("factorial.py", 10, "factorial"), # recursive - ("factorial.py", 10, "factorial"), # deeper - ("main.py", 5, "main"), - ], - )] - ), - MockInterpreterInfo( - 0, - [MockThreadInfo( - 1, - [ - ("factorial.py", 10, "factorial"), - ("factorial.py", 10, "factorial"), # different depth - ("main.py", 5, "main"), - ], - )] - ), - ] - - for frames in recursive_frames: - collector.collect([frames]) - - # Should capture both call paths - self.assertEqual(len(collector.stack_counter), 2) - - # First path should be longer (deeper recursion) than the second - path_tuples = list(collector.stack_counter.keys()) - paths = [p[0] for p in path_tuples] # Extract just the call paths - lengths = [len(p) for p in paths] - self.assertNotEqual(lengths[0], lengths[1]) - - # Both should contain factorial calls - self.assertTrue(any(any(f[2] == "factorial" for f in p) for p in paths)) - - # Verify total occurrences via aggregation - factorial_key = ("factorial.py", 10, "factorial") - main_key = ("main.py", 5, "main") - - def total_occurrences(func): - total = 0 - for (path, thread_id), count in collector.stack_counter.items(): - total += sum(1 for f in path if f == func) * count - return total - - self.assertEqual(total_occurrences(factorial_key), 5) - self.assertEqual(total_occurrences(main_key), 2) - - -@requires_subprocess() -@skip_if_not_supported -class TestSampleProfilerIntegration(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.test_script = ''' -import time -import os - -def slow_fibonacci(n): - """Recursive fibonacci - should show up prominently in profiler.""" - if n <= 1: - return n - return slow_fibonacci(n-1) + slow_fibonacci(n-2) - -def cpu_intensive_work(): - """CPU intensive work that should show in profiler.""" - result = 0 - for i in range(10000): - result += i * i - if i % 100 == 0: - result = result % 1000000 - return result - -def medium_computation(): - """Medium complexity function.""" - result = 0 - for i in range(100): - result += i * i - return result - -def fast_loop(): - """Fast simple loop.""" - total = 0 - for i in range(50): - total += i - return total - -def nested_calls(): - """Test nested function calls.""" - def level1(): - def level2(): - return medium_computation() - return level2() - return level1() - -def main_loop(): - """Main test loop with different execution paths.""" - iteration = 0 - - while True: - iteration += 1 - - # Different execution paths - focus on CPU intensive work - if iteration % 3 == 0: - # Very CPU intensive - result = cpu_intensive_work() - elif iteration % 5 == 0: - # Expensive recursive operation - result = slow_fibonacci(12) - else: - # Medium operation - result = nested_calls() - - # No sleep - keep CPU busy - -if __name__ == "__main__": - main_loop() -''' - - def test_sampling_basic_functionality(self): - with ( - test_subprocess(self.test_script) as subproc, - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=2, - sample_interval_usec=1000, # 1ms - show_summary=False, - ) - except PermissionError: - self.skipTest("Insufficient permissions for remote profiling") - - output = captured_output.getvalue() - - # Basic checks on output - self.assertIn("Captured", output) - self.assertIn("samples", output) - self.assertIn("Profile Stats", output) - - # Should see some of our test functions - self.assertIn("slow_fibonacci", output) - - def test_sampling_with_pstats_export(self): - pstats_out = tempfile.NamedTemporaryFile( - suffix=".pstats", delete=False - ) - self.addCleanup(close_and_unlink, pstats_out) - - with test_subprocess(self.test_script) as subproc: - # Suppress profiler output when testing file export - with ( - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=1, - filename=pstats_out.name, - sample_interval_usec=10000, - ) - except PermissionError: - self.skipTest( - "Insufficient permissions for remote profiling" - ) - - # Verify file was created and contains valid data - self.assertTrue(os.path.exists(pstats_out.name)) - self.assertGreater(os.path.getsize(pstats_out.name), 0) - - # Try to load the stats file - with open(pstats_out.name, "rb") as f: - stats_data = marshal.load(f) - - # Should be a dictionary with the sampled marker - self.assertIsInstance(stats_data, dict) - self.assertIn(("__sampled__",), stats_data) - self.assertTrue(stats_data[("__sampled__",)]) - - # Should have some function data - function_entries = [ - k for k in stats_data.keys() if k != ("__sampled__",) - ] - self.assertGreater(len(function_entries), 0) - - def test_sampling_with_collapsed_export(self): - collapsed_file = tempfile.NamedTemporaryFile( - suffix=".txt", delete=False - ) - self.addCleanup(close_and_unlink, collapsed_file) - - with ( - test_subprocess(self.test_script) as subproc, - ): - # Suppress profiler output when testing file export - with ( - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=1, - filename=collapsed_file.name, - output_format="collapsed", - sample_interval_usec=10000, - ) - except PermissionError: - self.skipTest( - "Insufficient permissions for remote profiling" - ) - - # Verify file was created and contains valid data - self.assertTrue(os.path.exists(collapsed_file.name)) - self.assertGreater(os.path.getsize(collapsed_file.name), 0) - - # Check file format - with open(collapsed_file.name, "r") as f: - content = f.read() - - lines = content.strip().split("\n") - self.assertGreater(len(lines), 0) - - # Each line should have format: stack_trace count - for line in lines: - parts = line.rsplit(" ", 1) - self.assertEqual(len(parts), 2) - - stack_trace, count_str = parts - self.assertGreater(len(stack_trace), 0) - self.assertTrue(count_str.isdigit()) - self.assertGreater(int(count_str), 0) - - # Stack trace should contain semicolon-separated entries - if ";" in stack_trace: - stack_parts = stack_trace.split(";") - for part in stack_parts: - # Each part should be file:function:line - self.assertIn(":", part) - - def test_sampling_all_threads(self): - with ( - test_subprocess(self.test_script) as subproc, - # Suppress profiler output - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=1, - all_threads=True, - sample_interval_usec=10000, - show_summary=False, - ) - except PermissionError: - self.skipTest("Insufficient permissions for remote profiling") - - # Just verify that sampling completed without error - # We're not testing output format here - - def test_sample_target_script(self): - script_file = tempfile.NamedTemporaryFile(delete=False) - script_file.write(self.test_script.encode("utf-8")) - script_file.flush() - self.addCleanup(close_and_unlink, script_file) - - test_args = ["profiling.sampling.sample", "-d", "1", script_file.name] - - with ( - mock.patch("sys.argv", test_args), - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.main() - except PermissionError: - self.skipTest("Insufficient permissions for remote profiling") - - output = captured_output.getvalue() - - # Basic checks on output - self.assertIn("Captured", output) - self.assertIn("samples", output) - self.assertIn("Profile Stats", output) - - # Should see some of our test functions - self.assertIn("slow_fibonacci", output) - - def test_sample_target_module(self): - tempdir = tempfile.TemporaryDirectory(delete=False) - self.addCleanup(lambda x: shutil.rmtree(x), tempdir.name) - - module_path = os.path.join(tempdir.name, "test_module.py") - - with open(module_path, "w") as f: - f.write(self.test_script) - - test_args = ["profiling.sampling.sample", "-d", "1", "-m", "test_module"] - - with ( - mock.patch("sys.argv", test_args), - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - # Change to temp directory so subprocess can find the module - contextlib.chdir(tempdir.name), - ): - try: - profiling.sampling.sample.main() - except PermissionError: - self.skipTest("Insufficient permissions for remote profiling") - - output = captured_output.getvalue() - - # Basic checks on output - self.assertIn("Captured", output) - self.assertIn("samples", output) - self.assertIn("Profile Stats", output) - - # Should see some of our test functions - self.assertIn("slow_fibonacci", output) - - -@skip_if_not_supported -@unittest.skipIf( - sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, - "Test only runs on Linux with process_vm_readv support", -) -class TestSampleProfilerErrorHandling(unittest.TestCase): - def test_invalid_pid(self): - with self.assertRaises((OSError, RuntimeError)): - profiling.sampling.sample.sample(-1, duration_sec=1) - - def test_process_dies_during_sampling(self): - with test_subprocess("import time; time.sleep(0.5); exit()") as subproc: - with ( - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=2, # Longer than process lifetime - sample_interval_usec=50000, - ) - except PermissionError: - self.skipTest( - "Insufficient permissions for remote profiling" - ) - - output = captured_output.getvalue() - - self.assertIn("Error rate", output) - - def test_invalid_output_format(self): - with self.assertRaises(ValueError): - profiling.sampling.sample.sample( - os.getpid(), - duration_sec=1, - output_format="invalid_format", - ) - - def test_invalid_output_format_with_mocked_profiler(self): - """Test invalid output format with proper mocking to avoid permission issues.""" - with mock.patch( - "profiling.sampling.sample.SampleProfiler" - ) as mock_profiler_class: - mock_profiler = mock.MagicMock() - mock_profiler_class.return_value = mock_profiler - - with self.assertRaises(ValueError) as cm: - profiling.sampling.sample.sample( - 12345, - duration_sec=1, - output_format="unknown_format", - ) - - # Should raise ValueError with the invalid format name - self.assertIn( - "Invalid output format: unknown_format", str(cm.exception) - ) - - def test_is_process_running(self): - with test_subprocess("import time; time.sleep(1000)") as subproc: - try: - profiler = SampleProfiler(pid=subproc.process.pid, sample_interval_usec=1000, all_threads=False) - except PermissionError: - self.skipTest( - "Insufficient permissions to read the stack trace" - ) - self.assertTrue(profiler._is_process_running()) - self.assertIsNotNone(profiler.unwinder.get_stack_trace()) - subproc.process.kill() - subproc.process.wait() - self.assertRaises(ProcessLookupError, profiler.unwinder.get_stack_trace) - - # Exit the context manager to ensure the process is terminated - self.assertFalse(profiler._is_process_running()) - self.assertRaises(ProcessLookupError, profiler.unwinder.get_stack_trace) - - @unittest.skipUnless(sys.platform == "linux", "Only valid on Linux") - def test_esrch_signal_handling(self): - with test_subprocess("import time; time.sleep(1000)") as subproc: - try: - unwinder = _remote_debugging.RemoteUnwinder(subproc.process.pid) - except PermissionError: - self.skipTest( - "Insufficient permissions to read the stack trace" - ) - initial_trace = unwinder.get_stack_trace() - self.assertIsNotNone(initial_trace) - - subproc.process.kill() - - # Wait for the process to die and try to get another trace - subproc.process.wait() - - with self.assertRaises(ProcessLookupError): - unwinder.get_stack_trace() - - def test_valid_output_formats(self): - """Test that all valid output formats are accepted.""" - valid_formats = ["pstats", "collapsed", "flamegraph", "gecko"] - - tempdir = tempfile.TemporaryDirectory(delete=False) - self.addCleanup(shutil.rmtree, tempdir.name) - - - with (contextlib.chdir(tempdir.name), captured_stdout(), captured_stderr()): - for fmt in valid_formats: - try: - # This will likely fail with permissions, but the format should be valid - profiling.sampling.sample.sample( - os.getpid(), - duration_sec=0.1, - output_format=fmt, - filename=f"test_{fmt}.out", - ) - except (OSError, RuntimeError, PermissionError): - # Expected errors - we just want to test format validation - pass - - def test_script_error_treatment(self): - script_file = tempfile.NamedTemporaryFile("w", delete=False, suffix=".py") - script_file.write("open('nonexistent_file.txt')\n") - script_file.close() - self.addCleanup(os.unlink, script_file.name) - - result = subprocess.run( - [sys.executable, "-m", "profiling.sampling.sample", "-d", "1", script_file.name], - capture_output=True, - text=True, - ) - output = result.stdout + result.stderr - - if "PermissionError" in output: - self.skipTest("Insufficient permissions for remote profiling") - self.assertNotIn("Script file not found", output) - self.assertIn("No such file or directory: 'nonexistent_file.txt'", output) - - -class TestSampleProfilerCLI(unittest.TestCase): - def _setup_sync_mocks(self, mock_socket, mock_popen): - """Helper to set up socket and process mocks for coordinator tests.""" - # Mock the sync socket with context manager support - mock_sock_instance = mock.MagicMock() - mock_sock_instance.getsockname.return_value = ("127.0.0.1", 12345) - - # Mock the connection with context manager support - mock_conn = mock.MagicMock() - mock_conn.recv.return_value = b"ready" - mock_conn.__enter__.return_value = mock_conn - mock_conn.__exit__.return_value = None - - # Mock accept() to return (connection, address) and support indexing - mock_accept_result = mock.MagicMock() - mock_accept_result.__getitem__.return_value = mock_conn # [0] returns the connection - mock_sock_instance.accept.return_value = mock_accept_result - - # Mock socket with context manager support - mock_sock_instance.__enter__.return_value = mock_sock_instance - mock_sock_instance.__exit__.return_value = None - mock_socket.return_value = mock_sock_instance - - # Mock the subprocess - mock_process = mock.MagicMock() - mock_process.pid = 12345 - mock_process.poll.return_value = None - mock_popen.return_value = mock_process - return mock_process - - def _verify_coordinator_command(self, mock_popen, expected_target_args): - """Helper to verify the coordinator command was called correctly.""" - args, kwargs = mock_popen.call_args - coordinator_cmd = args[0] - self.assertEqual(coordinator_cmd[0], sys.executable) - self.assertEqual(coordinator_cmd[1], "-m") - self.assertEqual(coordinator_cmd[2], "profiling.sampling._sync_coordinator") - self.assertEqual(coordinator_cmd[3], "12345") # port - # cwd is coordinator_cmd[4] - self.assertEqual(coordinator_cmd[5:], expected_target_args) - - @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") - def test_cli_module_argument_parsing(self): - test_args = ["profiling.sampling.sample", "-m", "mymodule"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - mock.patch("subprocess.Popen") as mock_popen, - mock.patch("socket.socket") as mock_socket, - ): - self._setup_sync_mocks(mock_socket, mock_popen) - profiling.sampling.sample.main() - - self._verify_coordinator_command(mock_popen, ("-m", "mymodule")) - mock_sample.assert_called_once_with( - 12345, - sort=2, # default sort (sort_value from args.sort) - sample_interval_usec=100, - duration_sec=10, - filename=None, - all_threads=False, - limit=15, - show_summary=True, - output_format="pstats", - realtime_stats=False, - mode=0, - native=False, - gc=True, - ) - - @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") - def test_cli_module_with_arguments(self): - test_args = ["profiling.sampling.sample", "-m", "mymodule", "arg1", "arg2", "--flag"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - mock.patch("subprocess.Popen") as mock_popen, - mock.patch("socket.socket") as mock_socket, - ): - self._setup_sync_mocks(mock_socket, mock_popen) - profiling.sampling.sample.main() - - self._verify_coordinator_command(mock_popen, ("-m", "mymodule", "arg1", "arg2", "--flag")) - mock_sample.assert_called_once_with( - 12345, - sort=2, - sample_interval_usec=100, - duration_sec=10, - filename=None, - all_threads=False, - limit=15, - show_summary=True, - output_format="pstats", - realtime_stats=False, - mode=0, - native=False, - gc=True, - ) - - @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") - def test_cli_script_argument_parsing(self): - test_args = ["profiling.sampling.sample", "myscript.py"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - mock.patch("subprocess.Popen") as mock_popen, - mock.patch("socket.socket") as mock_socket, - ): - self._setup_sync_mocks(mock_socket, mock_popen) - profiling.sampling.sample.main() - - self._verify_coordinator_command(mock_popen, ("myscript.py",)) - mock_sample.assert_called_once_with( - 12345, - sort=2, - sample_interval_usec=100, - duration_sec=10, - filename=None, - all_threads=False, - limit=15, - show_summary=True, - output_format="pstats", - realtime_stats=False, - mode=0, - native=False, - gc=True, - ) - - @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") - def test_cli_script_with_arguments(self): - test_args = ["profiling.sampling.sample", "myscript.py", "arg1", "arg2", "--flag"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - mock.patch("subprocess.Popen") as mock_popen, - mock.patch("socket.socket") as mock_socket, - ): - # Use the helper to set up mocks consistently - mock_process = self._setup_sync_mocks(mock_socket, mock_popen) - # Override specific behavior for this test - mock_process.wait.side_effect = [subprocess.TimeoutExpired(test_args, 0.1), None] - - profiling.sampling.sample.main() - - # Verify the coordinator command was called - args, kwargs = mock_popen.call_args - coordinator_cmd = args[0] - self.assertEqual(coordinator_cmd[0], sys.executable) - self.assertEqual(coordinator_cmd[1], "-m") - self.assertEqual(coordinator_cmd[2], "profiling.sampling._sync_coordinator") - self.assertEqual(coordinator_cmd[3], "12345") # port - # cwd is coordinator_cmd[4] - self.assertEqual(coordinator_cmd[5:], ("myscript.py", "arg1", "arg2", "--flag")) - - def test_cli_mutually_exclusive_pid_module(self): - test_args = ["profiling.sampling.sample", "-p", "12345", "-m", "mymodule"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("sys.stderr", io.StringIO()) as mock_stderr, - self.assertRaises(SystemExit) as cm, - ): - profiling.sampling.sample.main() - - self.assertEqual(cm.exception.code, 2) # argparse error - error_msg = mock_stderr.getvalue() - self.assertIn("not allowed with argument", error_msg) - - def test_cli_mutually_exclusive_pid_script(self): - test_args = ["profiling.sampling.sample", "-p", "12345", "myscript.py"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("sys.stderr", io.StringIO()) as mock_stderr, - self.assertRaises(SystemExit) as cm, - ): - profiling.sampling.sample.main() - - self.assertEqual(cm.exception.code, 2) # argparse error - error_msg = mock_stderr.getvalue() - self.assertIn("only one target type can be specified", error_msg) - - def test_cli_no_target_specified(self): - test_args = ["profiling.sampling.sample", "-d", "5"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("sys.stderr", io.StringIO()) as mock_stderr, - self.assertRaises(SystemExit) as cm, - ): - profiling.sampling.sample.main() - - self.assertEqual(cm.exception.code, 2) # argparse error - error_msg = mock_stderr.getvalue() - self.assertIn("one of the arguments", error_msg) - - @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") - def test_cli_module_with_profiler_options(self): - test_args = [ - "profiling.sampling.sample", "-i", "1000", "-d", "30", "-a", - "--sort-tottime", "-l", "20", "-m", "mymodule", - ] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - mock.patch("subprocess.Popen") as mock_popen, - mock.patch("socket.socket") as mock_socket, - ): - self._setup_sync_mocks(mock_socket, mock_popen) - profiling.sampling.sample.main() - - self._verify_coordinator_command(mock_popen, ("-m", "mymodule")) - mock_sample.assert_called_once_with( - 12345, - sort=1, # sort-tottime - sample_interval_usec=1000, - duration_sec=30, - filename=None, - all_threads=True, - limit=20, - show_summary=True, - output_format="pstats", - realtime_stats=False, - mode=0, - native=False, - gc=True, - ) - - @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") - def test_cli_script_with_profiler_options(self): - """Test script with various profiler options.""" - test_args = [ - "profiling.sampling.sample", "-i", "2000", "-d", "60", - "--collapsed", "-o", "output.txt", - "myscript.py", "scriptarg", - ] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - mock.patch("subprocess.Popen") as mock_popen, - mock.patch("socket.socket") as mock_socket, - ): - self._setup_sync_mocks(mock_socket, mock_popen) - profiling.sampling.sample.main() - - self._verify_coordinator_command(mock_popen, ("myscript.py", "scriptarg")) - # Verify profiler options were passed correctly - mock_sample.assert_called_once_with( - 12345, - sort=2, # default sort - sample_interval_usec=2000, - duration_sec=60, - filename="output.txt", - all_threads=False, - limit=15, - show_summary=True, - output_format="collapsed", - realtime_stats=False, - mode=0, - native=False, - gc=True, - ) - - def test_cli_empty_module_name(self): - test_args = ["profiling.sampling.sample", "-m"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("sys.stderr", io.StringIO()) as mock_stderr, - self.assertRaises(SystemExit) as cm, - ): - profiling.sampling.sample.main() - - self.assertEqual(cm.exception.code, 2) # argparse error - error_msg = mock_stderr.getvalue() - self.assertIn("argument -m/--module: expected one argument", error_msg) - - @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") - def test_cli_long_module_option(self): - test_args = ["profiling.sampling.sample", "--module", "mymodule", "arg1"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - mock.patch("subprocess.Popen") as mock_popen, - mock.patch("socket.socket") as mock_socket, - ): - self._setup_sync_mocks(mock_socket, mock_popen) - profiling.sampling.sample.main() - - self._verify_coordinator_command(mock_popen, ("-m", "mymodule", "arg1")) - - def test_cli_complex_script_arguments(self): - test_args = [ - "profiling.sampling.sample", "script.py", - "--input", "file.txt", "-v", "--output=/tmp/out", "positional" - ] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - mock.patch("profiling.sampling.sample._run_with_sync") as mock_run_with_sync, - ): - mock_process = mock.MagicMock() - mock_process.pid = 12345 - mock_process.wait.side_effect = [subprocess.TimeoutExpired(test_args, 0.1), None] - mock_process.poll.return_value = None - mock_run_with_sync.return_value = mock_process - - profiling.sampling.sample.main() - - mock_run_with_sync.assert_called_once_with(( - sys.executable, "script.py", - "--input", "file.txt", "-v", "--output=/tmp/out", "positional", - )) - - def test_cli_collapsed_format_validation(self): - """Test that CLI properly validates incompatible options with collapsed format.""" - test_cases = [ - # Test sort options are invalid with collapsed - ( - ["profiling.sampling.sample", "--collapsed", "--sort-nsamples", "-p", "12345"], - "sort", - ), - ( - ["profiling.sampling.sample", "--collapsed", "--sort-tottime", "-p", "12345"], - "sort", - ), - ( - [ - "profiling.sampling.sample", - "--collapsed", - "--sort-cumtime", - "-p", - "12345", - ], - "sort", - ), - ( - [ - "profiling.sampling.sample", - "--collapsed", - "--sort-sample-pct", - "-p", - "12345", - ], - "sort", - ), - ( - [ - "profiling.sampling.sample", - "--collapsed", - "--sort-cumul-pct", - "-p", - "12345", - ], - "sort", - ), - ( - ["profiling.sampling.sample", "--collapsed", "--sort-name", "-p", "12345"], - "sort", - ), - # Test limit option is invalid with collapsed - (["profiling.sampling.sample", "--collapsed", "-l", "20", "-p", "12345"], "limit"), - ( - ["profiling.sampling.sample", "--collapsed", "--limit", "20", "-p", "12345"], - "limit", - ), - # Test no-summary option is invalid with collapsed - ( - ["profiling.sampling.sample", "--collapsed", "--no-summary", "-p", "12345"], - "summary", - ), - ] - - for test_args, expected_error_keyword in test_cases: - with ( - mock.patch("sys.argv", test_args), - mock.patch("sys.stderr", io.StringIO()) as mock_stderr, - self.assertRaises(SystemExit) as cm, - ): - profiling.sampling.sample.main() - - self.assertEqual(cm.exception.code, 2) # argparse error code - error_msg = mock_stderr.getvalue() - self.assertIn("error:", error_msg) - self.assertIn("--pstats format", error_msg) - - def test_cli_default_collapsed_filename(self): - """Test that collapsed format gets a default filename when not specified.""" - test_args = ["profiling.sampling.sample", "--collapsed", "-p", "12345"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - ): - profiling.sampling.sample.main() - - # Check that filename was set to default collapsed format - mock_sample.assert_called_once() - call_args = mock_sample.call_args[1] - self.assertEqual(call_args["output_format"], "collapsed") - self.assertEqual(call_args["filename"], "collapsed.12345.txt") - - def test_cli_custom_output_filenames(self): - """Test custom output filenames for both formats.""" - test_cases = [ - ( - ["profiling.sampling.sample", "--pstats", "-o", "custom.pstats", "-p", "12345"], - "custom.pstats", - "pstats", - ), - ( - ["profiling.sampling.sample", "--collapsed", "-o", "custom.txt", "-p", "12345"], - "custom.txt", - "collapsed", - ), - ] - - for test_args, expected_filename, expected_format in test_cases: - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - ): - profiling.sampling.sample.main() - - mock_sample.assert_called_once() - call_args = mock_sample.call_args[1] - self.assertEqual(call_args["filename"], expected_filename) - self.assertEqual(call_args["output_format"], expected_format) - - def test_cli_missing_required_arguments(self): - """Test that CLI requires PID argument.""" - with ( - mock.patch("sys.argv", ["profiling.sampling.sample"]), - mock.patch("sys.stderr", io.StringIO()), - ): - with self.assertRaises(SystemExit): - profiling.sampling.sample.main() - - def test_cli_mutually_exclusive_format_options(self): - """Test that pstats and collapsed options are mutually exclusive.""" - with ( - mock.patch( - "sys.argv", - ["profiling.sampling.sample", "--pstats", "--collapsed", "-p", "12345"], - ), - mock.patch("sys.stderr", io.StringIO()), - ): - with self.assertRaises(SystemExit): - profiling.sampling.sample.main() - - def test_argument_parsing_basic(self): - test_args = ["profiling.sampling.sample", "-p", "12345"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - ): - profiling.sampling.sample.main() - - mock_sample.assert_called_once_with( - 12345, - sample_interval_usec=100, - duration_sec=10, - filename=None, - all_threads=False, - limit=15, - sort=2, - show_summary=True, - output_format="pstats", - realtime_stats=False, - mode=0, - native=False, - gc=True, - ) - - def test_sort_options(self): - sort_options = [ - ("--sort-nsamples", 0), - ("--sort-tottime", 1), - ("--sort-cumtime", 2), - ("--sort-sample-pct", 3), - ("--sort-cumul-pct", 4), - ("--sort-name", -1), - ] - - for option, expected_sort_value in sort_options: - test_args = ["profiling.sampling.sample", option, "-p", "12345"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - ): - profiling.sampling.sample.main() - - mock_sample.assert_called_once() - call_args = mock_sample.call_args[1] - self.assertEqual( - call_args["sort"], - expected_sort_value, - ) - mock_sample.reset_mock() - - -class TestCpuModeFiltering(unittest.TestCase): - """Test CPU mode filtering functionality (--mode=cpu).""" - - def test_mode_validation(self): - """Test that CLI validates mode choices correctly.""" - # Invalid mode choice should raise SystemExit - test_args = ["profiling.sampling.sample", "--mode", "invalid", "-p", "12345"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("sys.stderr", io.StringIO()) as mock_stderr, - self.assertRaises(SystemExit) as cm, - ): - profiling.sampling.sample.main() - - self.assertEqual(cm.exception.code, 2) # argparse error - error_msg = mock_stderr.getvalue() - self.assertIn("invalid choice", error_msg) - - def test_frames_filtered_with_skip_idle(self): - """Test that frames are actually filtered when skip_idle=True.""" - # Import thread status flags - try: - from _remote_debugging import THREAD_STATUS_HAS_GIL, THREAD_STATUS_ON_CPU - except ImportError: - THREAD_STATUS_HAS_GIL = (1 << 0) - THREAD_STATUS_ON_CPU = (1 << 1) - - # Create mock frames with different thread statuses - class MockThreadInfoWithStatus: - def __init__(self, thread_id, frame_info, status): - self.thread_id = thread_id - self.frame_info = frame_info - self.status = status - - # Create test data: active thread (HAS_GIL | ON_CPU), idle thread (neither), and another active thread - ACTIVE_STATUS = THREAD_STATUS_HAS_GIL | THREAD_STATUS_ON_CPU # Has GIL and on CPU - IDLE_STATUS = 0 # Neither has GIL nor on CPU - - test_frames = [ - MockInterpreterInfo(0, [ - MockThreadInfoWithStatus(1, [MockFrameInfo("active1.py", 10, "active_func1")], ACTIVE_STATUS), - MockThreadInfoWithStatus(2, [MockFrameInfo("idle.py", 20, "idle_func")], IDLE_STATUS), - MockThreadInfoWithStatus(3, [MockFrameInfo("active2.py", 30, "active_func2")], ACTIVE_STATUS), - ]) - ] - - # Test with skip_idle=True - should only process running threads - collector_skip = PstatsCollector(sample_interval_usec=1000, skip_idle=True) - collector_skip.collect(test_frames) - - # Should only have functions from running threads (status 0) - active1_key = ("active1.py", 10, "active_func1") - active2_key = ("active2.py", 30, "active_func2") - idle_key = ("idle.py", 20, "idle_func") - - self.assertIn(active1_key, collector_skip.result) - self.assertIn(active2_key, collector_skip.result) - self.assertNotIn(idle_key, collector_skip.result) # Idle thread should be filtered out - - # Test with skip_idle=False - should process all threads - collector_no_skip = PstatsCollector(sample_interval_usec=1000, skip_idle=False) - collector_no_skip.collect(test_frames) - - # Should have functions from all threads - self.assertIn(active1_key, collector_no_skip.result) - self.assertIn(active2_key, collector_no_skip.result) - self.assertIn(idle_key, collector_no_skip.result) # Idle thread should be included - - @requires_subprocess() - def test_cpu_mode_integration_filtering(self): - """Integration test: CPU mode should only capture active threads, not idle ones.""" - # Script with one mostly-idle thread and one CPU-active thread - cpu_vs_idle_script = ''' -import time -import threading - -cpu_ready = threading.Event() - -def idle_worker(): - time.sleep(999999) - -def cpu_active_worker(): - cpu_ready.set() - x = 1 - while True: - x += 1 - -def main(): - # Start both threads - idle_thread = threading.Thread(target=idle_worker) - cpu_thread = threading.Thread(target=cpu_active_worker) - idle_thread.start() - cpu_thread.start() - - # Wait for CPU thread to be running, then signal test - cpu_ready.wait() - _test_sock.sendall(b"threads_ready") - - idle_thread.join() - cpu_thread.join() - -main() - -''' - with test_subprocess(cpu_vs_idle_script) as subproc: - # Wait for signal that threads are running - response = subproc.socket.recv(1024) - self.assertEqual(response, b"threads_ready") - - with ( - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=2.0, - sample_interval_usec=5000, - mode=1, # CPU mode - show_summary=False, - all_threads=True, - ) - except (PermissionError, RuntimeError) as e: - self.skipTest("Insufficient permissions for remote profiling") - - cpu_mode_output = captured_output.getvalue() - - # Test wall-clock mode (mode=0) - should capture both functions - with ( - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=2.0, - sample_interval_usec=5000, - mode=0, # Wall-clock mode - show_summary=False, - all_threads=True, - ) - except (PermissionError, RuntimeError) as e: - self.skipTest("Insufficient permissions for remote profiling") - - wall_mode_output = captured_output.getvalue() - - # Verify both modes captured samples - self.assertIn("Captured", cpu_mode_output) - self.assertIn("samples", cpu_mode_output) - self.assertIn("Captured", wall_mode_output) - self.assertIn("samples", wall_mode_output) - - # CPU mode should strongly favor cpu_active_worker over mostly_idle_worker - self.assertIn("cpu_active_worker", cpu_mode_output) - self.assertNotIn("idle_worker", cpu_mode_output) - - # Wall-clock mode should capture both types of work - self.assertIn("cpu_active_worker", wall_mode_output) - self.assertIn("idle_worker", wall_mode_output) - - def test_cpu_mode_with_no_samples(self): - """Test that CPU mode handles no samples gracefully when no samples are collected.""" - # Mock a collector that returns empty stats - mock_collector = mock.MagicMock() - mock_collector.stats = {} - mock_collector.create_stats = mock.MagicMock() - - with ( - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - mock.patch("profiling.sampling.sample.PstatsCollector", return_value=mock_collector), - mock.patch("profiling.sampling.sample.SampleProfiler") as mock_profiler_class, - ): - mock_profiler = mock.MagicMock() - mock_profiler_class.return_value = mock_profiler - - profiling.sampling.sample.sample( - 12345, # dummy PID - duration_sec=0.5, - sample_interval_usec=5000, - mode=1, # CPU mode - show_summary=False, - all_threads=True, - ) - - output = captured_output.getvalue() - - # Should see the "No samples were collected" message - self.assertIn("No samples were collected", output) - self.assertIn("CPU mode", output) - - -class TestGilModeFiltering(unittest.TestCase): - """Test GIL mode filtering functionality (--mode=gil).""" - - def test_gil_mode_validation(self): - """Test that CLI accepts gil mode choice correctly.""" - test_args = ["profiling.sampling.sample", "--mode", "gil", "-p", "12345"] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - ): - try: - profiling.sampling.sample.main() - except SystemExit: - pass # Expected due to invalid PID - - # Should have attempted to call sample with mode=2 (GIL mode) - mock_sample.assert_called_once() - call_args = mock_sample.call_args[1] - self.assertEqual(call_args["mode"], 2) # PROFILING_MODE_GIL - - def test_gil_mode_sample_function_call(self): - """Test that sample() function correctly uses GIL mode.""" - with ( - mock.patch("profiling.sampling.sample.SampleProfiler") as mock_profiler, - mock.patch("profiling.sampling.sample.PstatsCollector") as mock_collector, - ): - # Mock the profiler instance - mock_instance = mock.Mock() - mock_profiler.return_value = mock_instance - - # Mock the collector instance - mock_collector_instance = mock.Mock() - mock_collector.return_value = mock_collector_instance - - # Call sample with GIL mode and a filename to avoid pstats creation - profiling.sampling.sample.sample( - 12345, - mode=2, # PROFILING_MODE_GIL - duration_sec=1, - sample_interval_usec=1000, - filename="test_output.txt", - ) - - # Verify SampleProfiler was created with correct mode - mock_profiler.assert_called_once() - call_args = mock_profiler.call_args - self.assertEqual(call_args[1]['mode'], 2) # mode parameter - - # Verify profiler.sample was called - mock_instance.sample.assert_called_once() - - # Verify collector.export was called since we provided a filename - mock_collector_instance.export.assert_called_once_with("test_output.txt") - - def test_gil_mode_collector_configuration(self): - """Test that collectors are configured correctly for GIL mode.""" - with ( - mock.patch("profiling.sampling.sample.SampleProfiler") as mock_profiler, - mock.patch("profiling.sampling.sample.PstatsCollector") as mock_collector, - captured_stdout(), captured_stderr() - ): - # Mock the profiler instance - mock_instance = mock.Mock() - mock_profiler.return_value = mock_instance - - # Call sample with GIL mode - profiling.sampling.sample.sample( - 12345, - mode=2, # PROFILING_MODE_GIL - output_format="pstats", - ) - - # Verify collector was created with skip_idle=True (since mode != WALL) - mock_collector.assert_called_once() - call_args = mock_collector.call_args[1] - self.assertTrue(call_args['skip_idle']) - - def test_gil_mode_with_collapsed_format(self): - """Test GIL mode with collapsed stack format.""" - with ( - mock.patch("profiling.sampling.sample.SampleProfiler") as mock_profiler, - mock.patch("profiling.sampling.sample.CollapsedStackCollector") as mock_collector, - ): - # Mock the profiler instance - mock_instance = mock.Mock() - mock_profiler.return_value = mock_instance - - # Call sample with GIL mode and collapsed format - profiling.sampling.sample.sample( - 12345, - mode=2, # PROFILING_MODE_GIL - output_format="collapsed", - filename="test_output.txt", - ) - - # Verify collector was created with skip_idle=True - mock_collector.assert_called_once() - call_args = mock_collector.call_args[1] - self.assertTrue(call_args['skip_idle']) - - def test_gil_mode_cli_argument_parsing(self): - """Test CLI argument parsing for GIL mode with various options.""" - test_args = [ - "profiling.sampling.sample", - "--mode", "gil", - "--interval", "500", - "--duration", "5", - "-p", "12345" - ] - - with ( - mock.patch("sys.argv", test_args), - mock.patch("profiling.sampling.sample.sample") as mock_sample, - ): - try: - profiling.sampling.sample.main() - except SystemExit: - pass # Expected due to invalid PID - - # Verify all arguments were parsed correctly - mock_sample.assert_called_once() - call_args = mock_sample.call_args[1] - self.assertEqual(call_args["mode"], 2) # GIL mode - self.assertEqual(call_args["sample_interval_usec"], 500) - self.assertEqual(call_args["duration_sec"], 5) - - @requires_subprocess() - def test_gil_mode_integration_behavior(self): - """Integration test: GIL mode should capture GIL-holding threads.""" - # Create a test script with GIL-releasing operations - gil_test_script = ''' -import time -import threading - -gil_ready = threading.Event() - -def gil_releasing_work(): - time.sleep(999999) - -def gil_holding_work(): - gil_ready.set() - x = 1 - while True: - x += 1 - -def main(): - # Start both threads - idle_thread = threading.Thread(target=gil_releasing_work) - cpu_thread = threading.Thread(target=gil_holding_work) - idle_thread.start() - cpu_thread.start() - - # Wait for GIL-holding thread to be running, then signal test - gil_ready.wait() - _test_sock.sendall(b"threads_ready") - - idle_thread.join() - cpu_thread.join() - -main() -''' - with test_subprocess(gil_test_script) as subproc: - # Wait for signal that threads are running - response = subproc.socket.recv(1024) - self.assertEqual(response, b"threads_ready") - - with ( - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=2.0, - sample_interval_usec=5000, - mode=2, # GIL mode - show_summary=False, - all_threads=True, - ) - except (PermissionError, RuntimeError) as e: - self.skipTest("Insufficient permissions for remote profiling") - - gil_mode_output = captured_output.getvalue() - - # Test wall-clock mode for comparison - with ( - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=0.5, - sample_interval_usec=5000, - mode=0, # Wall-clock mode - show_summary=False, - all_threads=True, - ) - except (PermissionError, RuntimeError) as e: - self.skipTest("Insufficient permissions for remote profiling") - - wall_mode_output = captured_output.getvalue() - - # GIL mode should primarily capture GIL-holding work - # (Note: actual behavior depends on threading implementation) - self.assertIn("gil_holding_work", gil_mode_output) - - # Wall-clock mode should capture both types of work - self.assertIn("gil_holding_work", wall_mode_output) - - def test_mode_constants_are_defined(self): - """Test that all profiling mode constants are properly defined.""" - self.assertEqual(profiling.sampling.sample.PROFILING_MODE_WALL, 0) - self.assertEqual(profiling.sampling.sample.PROFILING_MODE_CPU, 1) - self.assertEqual(profiling.sampling.sample.PROFILING_MODE_GIL, 2) - - def test_parse_mode_function(self): - """Test the _parse_mode function with all valid modes.""" - self.assertEqual(profiling.sampling.sample._parse_mode("wall"), 0) - self.assertEqual(profiling.sampling.sample._parse_mode("cpu"), 1) - self.assertEqual(profiling.sampling.sample._parse_mode("gil"), 2) - - # Test invalid mode raises KeyError - with self.assertRaises(KeyError): - profiling.sampling.sample._parse_mode("invalid") - - -@requires_subprocess() -@skip_if_not_supported -class TestGCFrameTracking(unittest.TestCase): - """Tests for GC frame tracking in the sampling profiler.""" - - @classmethod - def setUpClass(cls): - """Create a static test script with GC frames and CPU-intensive work.""" - cls.gc_test_script = ''' -import gc - -class ExpensiveGarbage: - """Class that triggers GC with expensive finalizer (callback).""" - def __init__(self): - self.cycle = self - - def __del__(self): - # CPU-intensive work in the finalizer callback - result = 0 - for i in range(100000): - result += i * i - if i % 1000 == 0: - result = result % 1000000 - -def main_loop(): - """Main loop that triggers GC with expensive callback.""" - while True: - ExpensiveGarbage() - gc.collect() - -if __name__ == "__main__": - main_loop() -''' - - def test_gc_frames_enabled(self): - """Test that GC frames appear when gc tracking is enabled.""" - with ( - test_subprocess(self.gc_test_script) as subproc, - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=1, - sample_interval_usec=5000, - show_summary=False, - native=False, - gc=True, - ) - except PermissionError: - self.skipTest("Insufficient permissions for remote profiling") - - output = captured_output.getvalue() - - # Should capture samples - self.assertIn("Captured", output) - self.assertIn("samples", output) - - # GC frames should be present - self.assertIn("", output) - - def test_gc_frames_disabled(self): - """Test that GC frames do not appear when gc tracking is disabled.""" - with ( - test_subprocess(self.gc_test_script) as subproc, - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=1, - sample_interval_usec=5000, - show_summary=False, - native=False, - gc=False, - ) - except PermissionError: - self.skipTest("Insufficient permissions for remote profiling") - - output = captured_output.getvalue() - - # Should capture samples - self.assertIn("Captured", output) - self.assertIn("samples", output) - - # GC frames should NOT be present - self.assertNotIn("", output) - - -@requires_subprocess() -@skip_if_not_supported -class TestNativeFrameTracking(unittest.TestCase): - """Tests for native frame tracking in the sampling profiler.""" - - @classmethod - def setUpClass(cls): - """Create a static test script with native frames and CPU-intensive work.""" - cls.native_test_script = ''' -import operator - -def main_loop(): - while True: - # Native code in the middle of the stack: - operator.call(inner) - -def inner(): - # Python code at the top of the stack: - for _ in range(1_000_0000): - pass - -if __name__ == "__main__": - main_loop() -''' - - def test_native_frames_enabled(self): - """Test that native frames appear when native tracking is enabled.""" - collapsed_file = tempfile.NamedTemporaryFile( - suffix=".txt", delete=False - ) - self.addCleanup(close_and_unlink, collapsed_file) - - with ( - test_subprocess(self.native_test_script) as subproc, - ): - # Suppress profiler output when testing file export - with ( - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=1, - filename=collapsed_file.name, - output_format="collapsed", - sample_interval_usec=1000, - native=True, - ) - except PermissionError: - self.skipTest("Insufficient permissions for remote profiling") - - # Verify file was created and contains valid data - self.assertTrue(os.path.exists(collapsed_file.name)) - self.assertGreater(os.path.getsize(collapsed_file.name), 0) - - # Check file format - with open(collapsed_file.name, "r") as f: - content = f.read() - - lines = content.strip().split("\n") - self.assertGreater(len(lines), 0) - - stacks = [line.rsplit(" ", 1)[0] for line in lines] - - # Most samples should have native code in the middle of the stack: - self.assertTrue(any(";;" in stack for stack in stacks)) - - # No samples should have native code at the top of the stack: - self.assertFalse(any(stack.endswith(";") for stack in stacks)) - - def test_native_frames_disabled(self): - """Test that native frames do not appear when native tracking is disabled.""" - with ( - test_subprocess(self.native_test_script) as subproc, - io.StringIO() as captured_output, - mock.patch("sys.stdout", captured_output), - ): - try: - profiling.sampling.sample.sample( - subproc.process.pid, - duration_sec=1, - sample_interval_usec=5000, - show_summary=False, - ) - except PermissionError: - self.skipTest("Insufficient permissions for remote profiling") - output = captured_output.getvalue() - # Native frames should NOT be present: - self.assertNotIn("", output) - - -@requires_subprocess() -@skip_if_not_supported -class TestProcessPoolExecutorSupport(unittest.TestCase): - """ - Test that ProcessPoolExecutor works correctly with profiling.sampling. - """ - - def test_process_pool_executor_pickle(self): - # gh-140729: test use ProcessPoolExecutor.map() can sampling - test_script = ''' -import concurrent.futures - -def worker(x): - return x * 2 - -if __name__ == "__main__": - with concurrent.futures.ProcessPoolExecutor() as executor: - results = list(executor.map(worker, [1, 2, 3])) - print(f"Results: {results}") -''' - with os_helper.temp_dir() as temp_dir: - script = script_helper.make_script( - temp_dir, 'test_process_pool_executor_pickle', test_script - ) - with SuppressCrashReport(): - with script_helper.spawn_python( - "-m", "profiling.sampling.sample", - "-d", "5", - "-i", "100000", - script, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True - ) as proc: - try: - stdout, stderr = proc.communicate(timeout=SHORT_TIMEOUT) - except subprocess.TimeoutExpired: - proc.kill() - stdout, stderr = proc.communicate() - - if "PermissionError" in stderr: - self.skipTest("Insufficient permissions for remote profiling") - - self.assertIn("Results: [2, 4, 6]", stdout) - self.assertNotIn("Can't pickle", stderr) -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_profiling/test_sampling_profiler/__init__.py b/Lib/test/test_profiling/test_sampling_profiler/__init__.py new file mode 100644 index 00000000000000..616ae5b49f03ef --- /dev/null +++ b/Lib/test/test_profiling/test_sampling_profiler/__init__.py @@ -0,0 +1,9 @@ +"""Tests for the sampling profiler (profiling.sampling).""" + +import os +from test.support import load_package_tests + + +def load_tests(*args): + """Load all tests from this subpackage.""" + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_profiling/test_sampling_profiler/helpers.py b/Lib/test/test_profiling/test_sampling_profiler/helpers.py new file mode 100644 index 00000000000000..abd5a7377ad68e --- /dev/null +++ b/Lib/test/test_profiling/test_sampling_profiler/helpers.py @@ -0,0 +1,101 @@ +"""Helper utilities for sampling profiler tests.""" + +import contextlib +import socket +import subprocess +import sys +import unittest +from collections import namedtuple + +from test.support import SHORT_TIMEOUT +from test.support.socket_helper import find_unused_port +from test.support.os_helper import unlink + + +PROCESS_VM_READV_SUPPORTED = False + +try: + from _remote_debugging import PROCESS_VM_READV_SUPPORTED # noqa: F401 + import _remote_debugging # noqa: F401 +except ImportError: + raise unittest.SkipTest( + "Test only runs when _remote_debugging is available" + ) +else: + import profiling.sampling # noqa: F401 + from profiling.sampling.sample import SampleProfiler # noqa: F401 + + +skip_if_not_supported = unittest.skipIf( + ( + sys.platform != "darwin" + and sys.platform != "linux" + and sys.platform != "win32" + ), + "Test only runs on Linux, Windows and MacOS", +) + +SubprocessInfo = namedtuple("SubprocessInfo", ["process", "socket"]) + + +@contextlib.contextmanager +def test_subprocess(script): + """Context manager to create a test subprocess with socket synchronization. + + Args: + script: Python code to execute in the subprocess + + Yields: + SubprocessInfo: Named tuple with process and socket objects + """ + # Find an unused port for socket communication + port = find_unused_port() + + # Inject socket connection code at the beginning of the script + socket_code = f""" +import socket +_test_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +_test_sock.connect(('localhost', {port})) +_test_sock.sendall(b"ready") +""" + + # Combine socket code with user script + full_script = socket_code + script + + # Create server socket to wait for process to be ready + server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server_socket.bind(("localhost", port)) + server_socket.settimeout(SHORT_TIMEOUT) + server_socket.listen(1) + + proc = subprocess.Popen( + [sys.executable, "-c", full_script], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + client_socket = None + try: + # Wait for process to connect and send ready signal + client_socket, _ = server_socket.accept() + server_socket.close() + response = client_socket.recv(1024) + if response != b"ready": + raise RuntimeError( + f"Unexpected response from subprocess: {response}" + ) + + yield SubprocessInfo(proc, client_socket) + finally: + if client_socket is not None: + client_socket.close() + if proc.poll() is None: + proc.kill() + proc.wait() + + +def close_and_unlink(file): + """Close a file and unlink it from the filesystem.""" + file.close() + unlink(file.name) diff --git a/Lib/test/test_profiling/test_sampling_profiler/mocks.py b/Lib/test/test_profiling/test_sampling_profiler/mocks.py new file mode 100644 index 00000000000000..9f1cd5b83e0856 --- /dev/null +++ b/Lib/test/test_profiling/test_sampling_profiler/mocks.py @@ -0,0 +1,38 @@ +"""Mock classes for sampling profiler tests.""" + + +class MockFrameInfo: + """Mock FrameInfo for testing since the real one isn't accessible.""" + + def __init__(self, filename, lineno, funcname): + self.filename = filename + self.lineno = lineno + self.funcname = funcname + + def __repr__(self): + return f"MockFrameInfo(filename='{self.filename}', lineno={self.lineno}, funcname='{self.funcname}')" + + +class MockThreadInfo: + """Mock ThreadInfo for testing since the real one isn't accessible.""" + + def __init__( + self, thread_id, frame_info, status=0 + ): # Default to THREAD_STATE_RUNNING (0) + self.thread_id = thread_id + self.frame_info = frame_info + self.status = status + + def __repr__(self): + return f"MockThreadInfo(thread_id={self.thread_id}, frame_info={self.frame_info}, status={self.status})" + + +class MockInterpreterInfo: + """Mock InterpreterInfo for testing since the real one isn't accessible.""" + + def __init__(self, interpreter_id, threads): + self.interpreter_id = interpreter_id + self.threads = threads + + def __repr__(self): + return f"MockInterpreterInfo(interpreter_id={self.interpreter_id}, threads={self.threads})" diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_advanced.py b/Lib/test/test_profiling/test_sampling_profiler/test_advanced.py new file mode 100644 index 00000000000000..578fb51bc0c9ef --- /dev/null +++ b/Lib/test/test_profiling/test_sampling_profiler/test_advanced.py @@ -0,0 +1,264 @@ +"""Tests for advanced sampling profiler features (GC tracking, native frames, ProcessPoolExecutor support).""" + +import io +import os +import subprocess +import tempfile +import unittest +from unittest import mock + +try: + import _remote_debugging # noqa: F401 + import profiling.sampling + import profiling.sampling.sample +except ImportError: + raise unittest.SkipTest( + "Test only runs when _remote_debugging is available" + ) + +from test.support import ( + SHORT_TIMEOUT, + SuppressCrashReport, + os_helper, + requires_subprocess, + script_helper, +) + +from .helpers import close_and_unlink, skip_if_not_supported, test_subprocess + + +@requires_subprocess() +@skip_if_not_supported +class TestGCFrameTracking(unittest.TestCase): + """Tests for GC frame tracking in the sampling profiler.""" + + @classmethod + def setUpClass(cls): + """Create a static test script with GC frames and CPU-intensive work.""" + cls.gc_test_script = ''' +import gc + +class ExpensiveGarbage: + """Class that triggers GC with expensive finalizer (callback).""" + def __init__(self): + self.cycle = self + + def __del__(self): + # CPU-intensive work in the finalizer callback + result = 0 + for i in range(100000): + result += i * i + if i % 1000 == 0: + result = result % 1000000 + +def main_loop(): + """Main loop that triggers GC with expensive callback.""" + while True: + ExpensiveGarbage() + gc.collect() + +if __name__ == "__main__": + main_loop() +''' + + def test_gc_frames_enabled(self): + """Test that GC frames appear when gc tracking is enabled.""" + with ( + test_subprocess(self.gc_test_script) as subproc, + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + profiling.sampling.sample.sample( + subproc.process.pid, + duration_sec=1, + sample_interval_usec=5000, + show_summary=False, + native=False, + gc=True, + ) + except PermissionError: + self.skipTest("Insufficient permissions for remote profiling") + + output = captured_output.getvalue() + + # Should capture samples + self.assertIn("Captured", output) + self.assertIn("samples", output) + + # GC frames should be present + self.assertIn("", output) + + def test_gc_frames_disabled(self): + """Test that GC frames do not appear when gc tracking is disabled.""" + with ( + test_subprocess(self.gc_test_script) as subproc, + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + profiling.sampling.sample.sample( + subproc.process.pid, + duration_sec=1, + sample_interval_usec=5000, + show_summary=False, + native=False, + gc=False, + ) + except PermissionError: + self.skipTest("Insufficient permissions for remote profiling") + + output = captured_output.getvalue() + + # Should capture samples + self.assertIn("Captured", output) + self.assertIn("samples", output) + + # GC frames should NOT be present + self.assertNotIn("", output) + + +@requires_subprocess() +@skip_if_not_supported +class TestNativeFrameTracking(unittest.TestCase): + """Tests for native frame tracking in the sampling profiler.""" + + @classmethod + def setUpClass(cls): + """Create a static test script with native frames and CPU-intensive work.""" + cls.native_test_script = """ +import operator + +def main_loop(): + while True: + # Native code in the middle of the stack: + operator.call(inner) + +def inner(): + # Python code at the top of the stack: + for _ in range(1_000_0000): + pass + +if __name__ == "__main__": + main_loop() +""" + + def test_native_frames_enabled(self): + """Test that native frames appear when native tracking is enabled.""" + collapsed_file = tempfile.NamedTemporaryFile( + suffix=".txt", delete=False + ) + self.addCleanup(close_and_unlink, collapsed_file) + + with ( + test_subprocess(self.native_test_script) as subproc, + ): + # Suppress profiler output when testing file export + with ( + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + profiling.sampling.sample.sample( + subproc.process.pid, + duration_sec=1, + filename=collapsed_file.name, + output_format="collapsed", + sample_interval_usec=1000, + native=True, + ) + except PermissionError: + self.skipTest( + "Insufficient permissions for remote profiling" + ) + + # Verify file was created and contains valid data + self.assertTrue(os.path.exists(collapsed_file.name)) + self.assertGreater(os.path.getsize(collapsed_file.name), 0) + + # Check file format + with open(collapsed_file.name, "r") as f: + content = f.read() + + lines = content.strip().split("\n") + self.assertGreater(len(lines), 0) + + stacks = [line.rsplit(" ", 1)[0] for line in lines] + + # Most samples should have native code in the middle of the stack: + self.assertTrue(any(";;" in stack for stack in stacks)) + + # No samples should have native code at the top of the stack: + self.assertFalse(any(stack.endswith(";") for stack in stacks)) + + def test_native_frames_disabled(self): + """Test that native frames do not appear when native tracking is disabled.""" + with ( + test_subprocess(self.native_test_script) as subproc, + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + profiling.sampling.sample.sample( + subproc.process.pid, + duration_sec=1, + sample_interval_usec=5000, + show_summary=False, + ) + except PermissionError: + self.skipTest("Insufficient permissions for remote profiling") + output = captured_output.getvalue() + # Native frames should NOT be present: + self.assertNotIn("", output) + + +@requires_subprocess() +@skip_if_not_supported +class TestProcessPoolExecutorSupport(unittest.TestCase): + """ + Test that ProcessPoolExecutor works correctly with profiling.sampling. + """ + + def test_process_pool_executor_pickle(self): + # gh-140729: test use ProcessPoolExecutor.map() can sampling + test_script = """ +import concurrent.futures + +def worker(x): + return x * 2 + +if __name__ == "__main__": + with concurrent.futures.ProcessPoolExecutor() as executor: + results = list(executor.map(worker, [1, 2, 3])) + print(f"Results: {results}") +""" + with os_helper.temp_dir() as temp_dir: + script = script_helper.make_script( + temp_dir, "test_process_pool_executor_pickle", test_script + ) + with SuppressCrashReport(): + with script_helper.spawn_python( + "-m", + "profiling.sampling.sample", + "-d", + "5", + "-i", + "100000", + script, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) as proc: + try: + stdout, stderr = proc.communicate( + timeout=SHORT_TIMEOUT + ) + except subprocess.TimeoutExpired: + proc.kill() + stdout, stderr = proc.communicate() + + if "PermissionError" in stderr: + self.skipTest("Insufficient permissions for remote profiling") + + self.assertIn("Results: [2, 4, 6]", stdout) + self.assertNotIn("Can't pickle", stderr) diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_cli.py b/Lib/test/test_profiling/test_sampling_profiler/test_cli.py new file mode 100644 index 00000000000000..5833920d1b96f3 --- /dev/null +++ b/Lib/test/test_profiling/test_sampling_profiler/test_cli.py @@ -0,0 +1,664 @@ +"""Tests for sampling profiler CLI argument parsing and functionality.""" + +import io +import subprocess +import sys +import unittest +from unittest import mock + +try: + import _remote_debugging # noqa: F401 + import profiling.sampling + import profiling.sampling.sample +except ImportError: + raise unittest.SkipTest( + "Test only runs when _remote_debugging is available" + ) + +from test.support import is_emscripten + + +class TestSampleProfilerCLI(unittest.TestCase): + def _setup_sync_mocks(self, mock_socket, mock_popen): + """Helper to set up socket and process mocks for coordinator tests.""" + # Mock the sync socket with context manager support + mock_sock_instance = mock.MagicMock() + mock_sock_instance.getsockname.return_value = ("127.0.0.1", 12345) + + # Mock the connection with context manager support + mock_conn = mock.MagicMock() + mock_conn.recv.return_value = b"ready" + mock_conn.__enter__.return_value = mock_conn + mock_conn.__exit__.return_value = None + + # Mock accept() to return (connection, address) and support indexing + mock_accept_result = mock.MagicMock() + mock_accept_result.__getitem__.return_value = ( + mock_conn # [0] returns the connection + ) + mock_sock_instance.accept.return_value = mock_accept_result + + # Mock socket with context manager support + mock_sock_instance.__enter__.return_value = mock_sock_instance + mock_sock_instance.__exit__.return_value = None + mock_socket.return_value = mock_sock_instance + + # Mock the subprocess + mock_process = mock.MagicMock() + mock_process.pid = 12345 + mock_process.poll.return_value = None + mock_popen.return_value = mock_process + return mock_process + + def _verify_coordinator_command(self, mock_popen, expected_target_args): + """Helper to verify the coordinator command was called correctly.""" + args, kwargs = mock_popen.call_args + coordinator_cmd = args[0] + self.assertEqual(coordinator_cmd[0], sys.executable) + self.assertEqual(coordinator_cmd[1], "-m") + self.assertEqual( + coordinator_cmd[2], "profiling.sampling._sync_coordinator" + ) + self.assertEqual(coordinator_cmd[3], "12345") # port + # cwd is coordinator_cmd[4] + self.assertEqual(coordinator_cmd[5:], expected_target_args) + + @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") + def test_cli_module_argument_parsing(self): + test_args = ["profiling.sampling.sample", "-m", "mymodule"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.sample.sample") as mock_sample, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch("socket.socket") as mock_socket, + ): + self._setup_sync_mocks(mock_socket, mock_popen) + profiling.sampling.sample.main() + + self._verify_coordinator_command(mock_popen, ("-m", "mymodule")) + mock_sample.assert_called_once_with( + 12345, + sort=2, # default sort (sort_value from args.sort) + sample_interval_usec=100, + duration_sec=10, + filename=None, + all_threads=False, + limit=15, + show_summary=True, + output_format="pstats", + realtime_stats=False, + mode=0, + native=False, + gc=True, + ) + + @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") + def test_cli_module_with_arguments(self): + test_args = [ + "profiling.sampling.sample", + "-m", + "mymodule", + "arg1", + "arg2", + "--flag", + ] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.sample.sample") as mock_sample, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch("socket.socket") as mock_socket, + ): + self._setup_sync_mocks(mock_socket, mock_popen) + profiling.sampling.sample.main() + + self._verify_coordinator_command( + mock_popen, ("-m", "mymodule", "arg1", "arg2", "--flag") + ) + mock_sample.assert_called_once_with( + 12345, + sort=2, + sample_interval_usec=100, + duration_sec=10, + filename=None, + all_threads=False, + limit=15, + show_summary=True, + output_format="pstats", + realtime_stats=False, + mode=0, + native=False, + gc=True, + ) + + @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") + def test_cli_script_argument_parsing(self): + test_args = ["profiling.sampling.sample", "myscript.py"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.sample.sample") as mock_sample, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch("socket.socket") as mock_socket, + ): + self._setup_sync_mocks(mock_socket, mock_popen) + profiling.sampling.sample.main() + + self._verify_coordinator_command(mock_popen, ("myscript.py",)) + mock_sample.assert_called_once_with( + 12345, + sort=2, + sample_interval_usec=100, + duration_sec=10, + filename=None, + all_threads=False, + limit=15, + show_summary=True, + output_format="pstats", + realtime_stats=False, + mode=0, + native=False, + gc=True, + ) + + @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") + def test_cli_script_with_arguments(self): + test_args = [ + "profiling.sampling.sample", + "myscript.py", + "arg1", + "arg2", + "--flag", + ] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.sample.sample") as mock_sample, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch("socket.socket") as mock_socket, + ): + # Use the helper to set up mocks consistently + mock_process = self._setup_sync_mocks(mock_socket, mock_popen) + # Override specific behavior for this test + mock_process.wait.side_effect = [ + subprocess.TimeoutExpired(test_args, 0.1), + None, + ] + + profiling.sampling.sample.main() + + # Verify the coordinator command was called + args, kwargs = mock_popen.call_args + coordinator_cmd = args[0] + self.assertEqual(coordinator_cmd[0], sys.executable) + self.assertEqual(coordinator_cmd[1], "-m") + self.assertEqual( + coordinator_cmd[2], "profiling.sampling._sync_coordinator" + ) + self.assertEqual(coordinator_cmd[3], "12345") # port + # cwd is coordinator_cmd[4] + self.assertEqual( + coordinator_cmd[5:], ("myscript.py", "arg1", "arg2", "--flag") + ) + + def test_cli_mutually_exclusive_pid_module(self): + test_args = [ + "profiling.sampling.sample", + "-p", + "12345", + "-m", + "mymodule", + ] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + self.assertRaises(SystemExit) as cm, + ): + profiling.sampling.sample.main() + + self.assertEqual(cm.exception.code, 2) # argparse error + error_msg = mock_stderr.getvalue() + self.assertIn("not allowed with argument", error_msg) + + def test_cli_mutually_exclusive_pid_script(self): + test_args = ["profiling.sampling.sample", "-p", "12345", "myscript.py"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + self.assertRaises(SystemExit) as cm, + ): + profiling.sampling.sample.main() + + self.assertEqual(cm.exception.code, 2) # argparse error + error_msg = mock_stderr.getvalue() + self.assertIn("only one target type can be specified", error_msg) + + def test_cli_no_target_specified(self): + test_args = ["profiling.sampling.sample", "-d", "5"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + self.assertRaises(SystemExit) as cm, + ): + profiling.sampling.sample.main() + + self.assertEqual(cm.exception.code, 2) # argparse error + error_msg = mock_stderr.getvalue() + self.assertIn("one of the arguments", error_msg) + + @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") + def test_cli_module_with_profiler_options(self): + test_args = [ + "profiling.sampling.sample", + "-i", + "1000", + "-d", + "30", + "-a", + "--sort-tottime", + "-l", + "20", + "-m", + "mymodule", + ] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.sample.sample") as mock_sample, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch("socket.socket") as mock_socket, + ): + self._setup_sync_mocks(mock_socket, mock_popen) + profiling.sampling.sample.main() + + self._verify_coordinator_command(mock_popen, ("-m", "mymodule")) + mock_sample.assert_called_once_with( + 12345, + sort=1, # sort-tottime + sample_interval_usec=1000, + duration_sec=30, + filename=None, + all_threads=True, + limit=20, + show_summary=True, + output_format="pstats", + realtime_stats=False, + mode=0, + native=False, + gc=True, + ) + + @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") + def test_cli_script_with_profiler_options(self): + """Test script with various profiler options.""" + test_args = [ + "profiling.sampling.sample", + "-i", + "2000", + "-d", + "60", + "--collapsed", + "-o", + "output.txt", + "myscript.py", + "scriptarg", + ] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.sample.sample") as mock_sample, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch("socket.socket") as mock_socket, + ): + self._setup_sync_mocks(mock_socket, mock_popen) + profiling.sampling.sample.main() + + self._verify_coordinator_command( + mock_popen, ("myscript.py", "scriptarg") + ) + # Verify profiler options were passed correctly + mock_sample.assert_called_once_with( + 12345, + sort=2, # default sort + sample_interval_usec=2000, + duration_sec=60, + filename="output.txt", + all_threads=False, + limit=15, + show_summary=True, + output_format="collapsed", + realtime_stats=False, + mode=0, + native=False, + gc=True, + ) + + def test_cli_empty_module_name(self): + test_args = ["profiling.sampling.sample", "-m"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + self.assertRaises(SystemExit) as cm, + ): + profiling.sampling.sample.main() + + self.assertEqual(cm.exception.code, 2) # argparse error + error_msg = mock_stderr.getvalue() + self.assertIn("argument -m/--module: expected one argument", error_msg) + + @unittest.skipIf(is_emscripten, "socket.SO_REUSEADDR does not exist") + def test_cli_long_module_option(self): + test_args = [ + "profiling.sampling.sample", + "--module", + "mymodule", + "arg1", + ] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.sample.sample") as mock_sample, + mock.patch("subprocess.Popen") as mock_popen, + mock.patch("socket.socket") as mock_socket, + ): + self._setup_sync_mocks(mock_socket, mock_popen) + profiling.sampling.sample.main() + + self._verify_coordinator_command( + mock_popen, ("-m", "mymodule", "arg1") + ) + + def test_cli_complex_script_arguments(self): + test_args = [ + "profiling.sampling.sample", + "script.py", + "--input", + "file.txt", + "-v", + "--output=/tmp/out", + "positional", + ] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.sample.sample") as mock_sample, + mock.patch( + "profiling.sampling.sample._run_with_sync" + ) as mock_run_with_sync, + ): + mock_process = mock.MagicMock() + mock_process.pid = 12345 + mock_process.wait.side_effect = [ + subprocess.TimeoutExpired(test_args, 0.1), + None, + ] + mock_process.poll.return_value = None + mock_run_with_sync.return_value = mock_process + + profiling.sampling.sample.main() + + mock_run_with_sync.assert_called_once_with( + ( + sys.executable, + "script.py", + "--input", + "file.txt", + "-v", + "--output=/tmp/out", + "positional", + ) + ) + + def test_cli_collapsed_format_validation(self): + """Test that CLI properly validates incompatible options with collapsed format.""" + test_cases = [ + # Test sort options are invalid with collapsed + ( + [ + "profiling.sampling.sample", + "--collapsed", + "--sort-nsamples", + "-p", + "12345", + ], + "sort", + ), + ( + [ + "profiling.sampling.sample", + "--collapsed", + "--sort-tottime", + "-p", + "12345", + ], + "sort", + ), + ( + [ + "profiling.sampling.sample", + "--collapsed", + "--sort-cumtime", + "-p", + "12345", + ], + "sort", + ), + ( + [ + "profiling.sampling.sample", + "--collapsed", + "--sort-sample-pct", + "-p", + "12345", + ], + "sort", + ), + ( + [ + "profiling.sampling.sample", + "--collapsed", + "--sort-cumul-pct", + "-p", + "12345", + ], + "sort", + ), + ( + [ + "profiling.sampling.sample", + "--collapsed", + "--sort-name", + "-p", + "12345", + ], + "sort", + ), + # Test limit option is invalid with collapsed + ( + [ + "profiling.sampling.sample", + "--collapsed", + "-l", + "20", + "-p", + "12345", + ], + "limit", + ), + ( + [ + "profiling.sampling.sample", + "--collapsed", + "--limit", + "20", + "-p", + "12345", + ], + "limit", + ), + # Test no-summary option is invalid with collapsed + ( + [ + "profiling.sampling.sample", + "--collapsed", + "--no-summary", + "-p", + "12345", + ], + "summary", + ), + ] + + for test_args, expected_error_keyword in test_cases: + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + self.assertRaises(SystemExit) as cm, + ): + profiling.sampling.sample.main() + + self.assertEqual(cm.exception.code, 2) # argparse error code + error_msg = mock_stderr.getvalue() + self.assertIn("error:", error_msg) + self.assertIn("--pstats format", error_msg) + + def test_cli_default_collapsed_filename(self): + """Test that collapsed format gets a default filename when not specified.""" + test_args = ["profiling.sampling.sample", "--collapsed", "-p", "12345"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.sample.sample") as mock_sample, + ): + profiling.sampling.sample.main() + + # Check that filename was set to default collapsed format + mock_sample.assert_called_once() + call_args = mock_sample.call_args[1] + self.assertEqual(call_args["output_format"], "collapsed") + self.assertEqual(call_args["filename"], "collapsed.12345.txt") + + def test_cli_custom_output_filenames(self): + """Test custom output filenames for both formats.""" + test_cases = [ + ( + [ + "profiling.sampling.sample", + "--pstats", + "-o", + "custom.pstats", + "-p", + "12345", + ], + "custom.pstats", + "pstats", + ), + ( + [ + "profiling.sampling.sample", + "--collapsed", + "-o", + "custom.txt", + "-p", + "12345", + ], + "custom.txt", + "collapsed", + ), + ] + + for test_args, expected_filename, expected_format in test_cases: + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.sample.sample") as mock_sample, + ): + profiling.sampling.sample.main() + + mock_sample.assert_called_once() + call_args = mock_sample.call_args[1] + self.assertEqual(call_args["filename"], expected_filename) + self.assertEqual(call_args["output_format"], expected_format) + + def test_cli_missing_required_arguments(self): + """Test that CLI requires PID argument.""" + with ( + mock.patch("sys.argv", ["profiling.sampling.sample"]), + mock.patch("sys.stderr", io.StringIO()), + ): + with self.assertRaises(SystemExit): + profiling.sampling.sample.main() + + def test_cli_mutually_exclusive_format_options(self): + """Test that pstats and collapsed options are mutually exclusive.""" + with ( + mock.patch( + "sys.argv", + [ + "profiling.sampling.sample", + "--pstats", + "--collapsed", + "-p", + "12345", + ], + ), + mock.patch("sys.stderr", io.StringIO()), + ): + with self.assertRaises(SystemExit): + profiling.sampling.sample.main() + + def test_argument_parsing_basic(self): + test_args = ["profiling.sampling.sample", "-p", "12345"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.sample.sample") as mock_sample, + ): + profiling.sampling.sample.main() + + mock_sample.assert_called_once_with( + 12345, + sample_interval_usec=100, + duration_sec=10, + filename=None, + all_threads=False, + limit=15, + sort=2, + show_summary=True, + output_format="pstats", + realtime_stats=False, + mode=0, + native=False, + gc=True, + ) + + def test_sort_options(self): + sort_options = [ + ("--sort-nsamples", 0), + ("--sort-tottime", 1), + ("--sort-cumtime", 2), + ("--sort-sample-pct", 3), + ("--sort-cumul-pct", 4), + ("--sort-name", -1), + ] + + for option, expected_sort_value in sort_options: + test_args = ["profiling.sampling.sample", option, "-p", "12345"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.sample.sample") as mock_sample, + ): + profiling.sampling.sample.main() + + mock_sample.assert_called_once() + call_args = mock_sample.call_args[1] + self.assertEqual( + call_args["sort"], + expected_sort_value, + ) + mock_sample.reset_mock() diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py new file mode 100644 index 00000000000000..4a24256203c187 --- /dev/null +++ b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py @@ -0,0 +1,896 @@ +"""Tests for sampling profiler collector components.""" + +import json +import marshal +import os +import tempfile +import unittest + +try: + import _remote_debugging # noqa: F401 + from profiling.sampling.pstats_collector import PstatsCollector + from profiling.sampling.stack_collector import ( + CollapsedStackCollector, + FlamegraphCollector, + ) + from profiling.sampling.gecko_collector import GeckoCollector +except ImportError: + raise unittest.SkipTest( + "Test only runs when _remote_debugging is available" + ) + +from test.support import captured_stdout, captured_stderr + +from .mocks import MockFrameInfo, MockThreadInfo, MockInterpreterInfo +from .helpers import close_and_unlink + + +class TestSampleProfilerComponents(unittest.TestCase): + """Unit tests for individual profiler components.""" + + def test_mock_frame_info_with_empty_and_unicode_values(self): + """Test MockFrameInfo handles empty strings, unicode characters, and very long names correctly.""" + # Test with empty strings + frame = MockFrameInfo("", 0, "") + self.assertEqual(frame.filename, "") + self.assertEqual(frame.lineno, 0) + self.assertEqual(frame.funcname, "") + self.assertIn("filename=''", repr(frame)) + + # Test with unicode characters + frame = MockFrameInfo("文件.py", 42, "函数名") + self.assertEqual(frame.filename, "文件.py") + self.assertEqual(frame.funcname, "函数名") + + # Test with very long names + long_filename = "x" * 1000 + ".py" + long_funcname = "func_" + "x" * 1000 + frame = MockFrameInfo(long_filename, 999999, long_funcname) + self.assertEqual(frame.filename, long_filename) + self.assertEqual(frame.lineno, 999999) + self.assertEqual(frame.funcname, long_funcname) + + def test_pstats_collector_with_extreme_intervals_and_empty_data(self): + """Test PstatsCollector handles zero/large intervals, empty frames, None thread IDs, and duplicate frames.""" + # Test with zero interval + collector = PstatsCollector(sample_interval_usec=0) + self.assertEqual(collector.sample_interval_usec, 0) + + # Test with very large interval + collector = PstatsCollector(sample_interval_usec=1000000000) + self.assertEqual(collector.sample_interval_usec, 1000000000) + + # Test collecting empty frames list + collector = PstatsCollector(sample_interval_usec=1000) + collector.collect([]) + self.assertEqual(len(collector.result), 0) + + # Test collecting frames with None thread id + test_frames = [ + MockInterpreterInfo( + 0, + [MockThreadInfo(None, [MockFrameInfo("file.py", 10, "func")])], + ) + ] + collector.collect(test_frames) + # Should still process the frames + self.assertEqual(len(collector.result), 1) + + # Test collecting duplicate frames in same sample + test_frames = [ + MockInterpreterInfo( + 0, # interpreter_id + [ + MockThreadInfo( + 1, + [ + MockFrameInfo("file.py", 10, "func1"), + MockFrameInfo("file.py", 10, "func1"), # Duplicate + ], + ) + ], + ) + ] + collector = PstatsCollector(sample_interval_usec=1000) + collector.collect(test_frames) + # Should count both occurrences + self.assertEqual( + collector.result[("file.py", 10, "func1")]["cumulative_calls"], 2 + ) + + def test_pstats_collector_single_frame_stacks(self): + """Test PstatsCollector with single-frame call stacks to trigger len(frames) <= 1 branch.""" + collector = PstatsCollector(sample_interval_usec=1000) + + # Test with exactly one frame (should trigger the <= 1 condition) + single_frame = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, [MockFrameInfo("single.py", 10, "single_func")] + ) + ], + ) + ] + collector.collect(single_frame) + + # Should record the single frame with inline call + self.assertEqual(len(collector.result), 1) + single_key = ("single.py", 10, "single_func") + self.assertIn(single_key, collector.result) + self.assertEqual(collector.result[single_key]["direct_calls"], 1) + self.assertEqual(collector.result[single_key]["cumulative_calls"], 1) + + # Test with empty frames (should also trigger <= 1 condition) + empty_frames = [MockInterpreterInfo(0, [MockThreadInfo(1, [])])] + collector.collect(empty_frames) + + # Should not add any new entries + self.assertEqual( + len(collector.result), 1 + ) # Still just the single frame + + # Test mixed single and multi-frame stacks + mixed_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [MockFrameInfo("single2.py", 20, "single_func2")], + ), # Single frame + MockThreadInfo( + 2, + [ # Multi-frame stack + MockFrameInfo("multi.py", 30, "multi_func1"), + MockFrameInfo("multi.py", 40, "multi_func2"), + ], + ), + ], + ), + ] + collector.collect(mixed_frames) + + # Should have recorded all functions + self.assertEqual( + len(collector.result), 4 + ) # single + single2 + multi1 + multi2 + + # Verify single frame handling + single2_key = ("single2.py", 20, "single_func2") + self.assertIn(single2_key, collector.result) + self.assertEqual(collector.result[single2_key]["direct_calls"], 1) + self.assertEqual(collector.result[single2_key]["cumulative_calls"], 1) + + # Verify multi-frame handling still works + multi1_key = ("multi.py", 30, "multi_func1") + multi2_key = ("multi.py", 40, "multi_func2") + self.assertIn(multi1_key, collector.result) + self.assertIn(multi2_key, collector.result) + self.assertEqual(collector.result[multi1_key]["direct_calls"], 1) + self.assertEqual( + collector.result[multi2_key]["cumulative_calls"], 1 + ) # Called from multi1 + + def test_collapsed_stack_collector_with_empty_and_deep_stacks(self): + """Test CollapsedStackCollector handles empty frames, single-frame stacks, and very deep call stacks.""" + collector = CollapsedStackCollector() + + # Test with empty frames + collector.collect([]) + self.assertEqual(len(collector.stack_counter), 0) + + # Test with single frame stack + test_frames = [ + MockInterpreterInfo( + 0, [MockThreadInfo(1, [("file.py", 10, "func")])] + ) + ] + collector.collect(test_frames) + self.assertEqual(len(collector.stack_counter), 1) + (((path, thread_id), count),) = collector.stack_counter.items() + self.assertEqual(path, (("file.py", 10, "func"),)) + self.assertEqual(thread_id, 1) + self.assertEqual(count, 1) + + # Test with very deep stack + deep_stack = [(f"file{i}.py", i, f"func{i}") for i in range(100)] + test_frames = [MockInterpreterInfo(0, [MockThreadInfo(1, deep_stack)])] + collector = CollapsedStackCollector() + collector.collect(test_frames) + # One aggregated path with 100 frames (reversed) + (((path_tuple, thread_id),),) = (collector.stack_counter.keys(),) + self.assertEqual(len(path_tuple), 100) + self.assertEqual(path_tuple[0], ("file99.py", 99, "func99")) + self.assertEqual(path_tuple[-1], ("file0.py", 0, "func0")) + self.assertEqual(thread_id, 1) + + def test_pstats_collector_basic(self): + """Test basic PstatsCollector functionality.""" + collector = PstatsCollector(sample_interval_usec=1000) + + # Test empty state + self.assertEqual(len(collector.result), 0) + self.assertEqual(len(collector.stats), 0) + + # Test collecting sample data + test_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [ + MockFrameInfo("file.py", 10, "func1"), + MockFrameInfo("file.py", 20, "func2"), + ], + ) + ], + ) + ] + collector.collect(test_frames) + + # Should have recorded calls for both functions + self.assertEqual(len(collector.result), 2) + self.assertIn(("file.py", 10, "func1"), collector.result) + self.assertIn(("file.py", 20, "func2"), collector.result) + + # Top-level function should have direct call + self.assertEqual( + collector.result[("file.py", 10, "func1")]["direct_calls"], 1 + ) + self.assertEqual( + collector.result[("file.py", 10, "func1")]["cumulative_calls"], 1 + ) + + # Calling function should have cumulative call but no direct calls + self.assertEqual( + collector.result[("file.py", 20, "func2")]["cumulative_calls"], 1 + ) + self.assertEqual( + collector.result[("file.py", 20, "func2")]["direct_calls"], 0 + ) + + def test_pstats_collector_create_stats(self): + """Test PstatsCollector stats creation.""" + collector = PstatsCollector( + sample_interval_usec=1000000 + ) # 1 second intervals + + test_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [ + MockFrameInfo("file.py", 10, "func1"), + MockFrameInfo("file.py", 20, "func2"), + ], + ) + ], + ) + ] + collector.collect(test_frames) + collector.collect(test_frames) # Collect twice + + collector.create_stats() + + # Check stats format: (direct_calls, cumulative_calls, tt, ct, callers) + func1_stats = collector.stats[("file.py", 10, "func1")] + self.assertEqual(func1_stats[0], 2) # direct_calls (top of stack) + self.assertEqual(func1_stats[1], 2) # cumulative_calls + self.assertEqual( + func1_stats[2], 2.0 + ) # tt (total time - 2 samples * 1 sec) + self.assertEqual(func1_stats[3], 2.0) # ct (cumulative time) + + func2_stats = collector.stats[("file.py", 20, "func2")] + self.assertEqual( + func2_stats[0], 0 + ) # direct_calls (never top of stack) + self.assertEqual( + func2_stats[1], 2 + ) # cumulative_calls (appears in stack) + self.assertEqual(func2_stats[2], 0.0) # tt (no direct calls) + self.assertEqual(func2_stats[3], 2.0) # ct (cumulative time) + + def test_collapsed_stack_collector_basic(self): + collector = CollapsedStackCollector() + + # Test empty state + self.assertEqual(len(collector.stack_counter), 0) + + # Test collecting sample data + test_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, [("file.py", 10, "func1"), ("file.py", 20, "func2")] + ) + ], + ) + ] + collector.collect(test_frames) + + # Should store one reversed path + self.assertEqual(len(collector.stack_counter), 1) + (((path, thread_id), count),) = collector.stack_counter.items() + expected_tree = (("file.py", 20, "func2"), ("file.py", 10, "func1")) + self.assertEqual(path, expected_tree) + self.assertEqual(thread_id, 1) + self.assertEqual(count, 1) + + def test_collapsed_stack_collector_export(self): + collapsed_out = tempfile.NamedTemporaryFile(delete=False) + self.addCleanup(close_and_unlink, collapsed_out) + + collector = CollapsedStackCollector() + + test_frames1 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, [("file.py", 10, "func1"), ("file.py", 20, "func2")] + ) + ], + ) + ] + test_frames2 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, [("file.py", 10, "func1"), ("file.py", 20, "func2")] + ) + ], + ) + ] # Same stack + test_frames3 = [ + MockInterpreterInfo( + 0, [MockThreadInfo(1, [("other.py", 5, "other_func")])] + ) + ] + + collector.collect(test_frames1) + collector.collect(test_frames2) + collector.collect(test_frames3) + + with captured_stdout(), captured_stderr(): + collector.export(collapsed_out.name) + # Check file contents + with open(collapsed_out.name, "r") as f: + content = f.read() + + lines = content.strip().split("\n") + self.assertEqual(len(lines), 2) # Two unique stacks + + # Check collapsed format: tid:X;file:func:line;file:func:line count + stack1_expected = "tid:1;file.py:func2:20;file.py:func1:10 2" + stack2_expected = "tid:1;other.py:other_func:5 1" + + self.assertIn(stack1_expected, lines) + self.assertIn(stack2_expected, lines) + + def test_flamegraph_collector_basic(self): + """Test basic FlamegraphCollector functionality.""" + collector = FlamegraphCollector() + + # Empty collector should produce 'No Data' + data = collector._convert_to_flamegraph_format() + # With string table, name is now an index - resolve it using the strings array + strings = data.get("strings", []) + name_index = data.get("name", 0) + resolved_name = ( + strings[name_index] + if isinstance(name_index, int) and 0 <= name_index < len(strings) + else str(name_index) + ) + self.assertIn(resolved_name, ("No Data", "No significant data")) + + # Test collecting sample data + test_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, [("file.py", 10, "func1"), ("file.py", 20, "func2")] + ) + ], + ) + ] + collector.collect(test_frames) + + # Convert and verify structure: func2 -> func1 with counts = 1 + data = collector._convert_to_flamegraph_format() + # Expect promotion: root is the single child (func2), with func1 as its only child + strings = data.get("strings", []) + name_index = data.get("name", 0) + name = ( + strings[name_index] + if isinstance(name_index, int) and 0 <= name_index < len(strings) + else str(name_index) + ) + self.assertIsInstance(name, str) + self.assertTrue(name.startswith("Program Root: ")) + self.assertIn("func2 (file.py:20)", name) # formatted name + children = data.get("children", []) + self.assertEqual(len(children), 1) + child = children[0] + child_name_index = child.get("name", 0) + child_name = ( + strings[child_name_index] + if isinstance(child_name_index, int) + and 0 <= child_name_index < len(strings) + else str(child_name_index) + ) + self.assertIn("func1 (file.py:10)", child_name) # formatted name + self.assertEqual(child["value"], 1) + + def test_flamegraph_collector_export(self): + """Test flamegraph HTML export functionality.""" + flamegraph_out = tempfile.NamedTemporaryFile( + suffix=".html", delete=False + ) + self.addCleanup(close_and_unlink, flamegraph_out) + + collector = FlamegraphCollector() + + # Create some test data (use Interpreter/Thread objects like runtime) + test_frames1 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, [("file.py", 10, "func1"), ("file.py", 20, "func2")] + ) + ], + ) + ] + test_frames2 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, [("file.py", 10, "func1"), ("file.py", 20, "func2")] + ) + ], + ) + ] # Same stack + test_frames3 = [ + MockInterpreterInfo( + 0, [MockThreadInfo(1, [("other.py", 5, "other_func")])] + ) + ] + + collector.collect(test_frames1) + collector.collect(test_frames2) + collector.collect(test_frames3) + + # Export flamegraph + with captured_stdout(), captured_stderr(): + collector.export(flamegraph_out.name) + + # Verify file was created and contains valid data + self.assertTrue(os.path.exists(flamegraph_out.name)) + self.assertGreater(os.path.getsize(flamegraph_out.name), 0) + + # Check file contains HTML content + with open(flamegraph_out.name, "r", encoding="utf-8") as f: + content = f.read() + + # Should be valid HTML + self.assertIn("", content.lower()) + self.assertIn(" 0) + self.assertGreater(mock_collector.collect.call_count, 0) + self.assertLessEqual(mock_collector.collect.call_count, 3) + + def test_sample_profiler_missed_samples_warning(self): + """Test that the profiler warns about missed samples when sampling is too slow.""" + + mock_unwinder = mock.MagicMock() + mock_unwinder.get_stack_trace.return_value = [ + ( + 1, + [ + mock.MagicMock( + filename="test.py", lineno=10, funcname="test_func" + ) + ], + ) + ] + + with mock.patch( + "_remote_debugging.RemoteUnwinder" + ) as mock_unwinder_class: + mock_unwinder_class.return_value = mock_unwinder + + # Use very short interval that we'll miss + profiler = SampleProfiler( + pid=12345, sample_interval_usec=1000, all_threads=False + ) # 1ms interval + + mock_collector = mock.MagicMock() + + # Simulate slow sampling where we miss many samples + times = [ + 0.0, + 0.1, + 0.2, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + ] # Extra time points to avoid StopIteration + + with mock.patch("time.perf_counter", side_effect=times): + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + profiler.sample(mock_collector, duration_sec=0.5) + + result = output.getvalue() + + # Should warn about missed samples + self.assertIn("Warning: missed", result) + self.assertIn("samples from the expected total", result) + + +@force_not_colorized_test_class +class TestPrintSampledStats(unittest.TestCase): + """Test the print_sampled_stats function.""" + + def setUp(self): + """Set up test data.""" + # Mock stats data + self.mock_stats = mock.MagicMock() + self.mock_stats.stats = { + ("file1.py", 10, "func1"): ( + 100, + 100, + 0.5, + 0.5, + {}, + ), # cc, nc, tt, ct, callers + ("file2.py", 20, "func2"): (50, 50, 0.25, 0.3, {}), + ("file3.py", 30, "func3"): (200, 200, 1.5, 2.0, {}), + ("file4.py", 40, "func4"): ( + 10, + 10, + 0.001, + 0.001, + {}, + ), # millisecond range + ("file5.py", 50, "func5"): ( + 5, + 5, + 0.000001, + 0.000002, + {}, + ), # microsecond range + } + + def test_print_sampled_stats_basic(self): + """Test basic print_sampled_stats functionality.""" + + # Capture output + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats(self.mock_stats, sample_interval_usec=100) + + result = output.getvalue() + + # Check header is present + self.assertIn("Profile Stats:", result) + self.assertIn("nsamples", result) + self.assertIn("tottime", result) + self.assertIn("cumtime", result) + + # Check functions are present + self.assertIn("func1", result) + self.assertIn("func2", result) + self.assertIn("func3", result) + + def test_print_sampled_stats_sorting(self): + """Test different sorting options.""" + + # Test sort by calls + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + self.mock_stats, sort=0, sample_interval_usec=100 + ) + + result = output.getvalue() + lines = result.strip().split("\n") + + # Find the data lines (skip header) + data_lines = [l for l in lines if "file" in l and ".py" in l] + # func3 should be first (200 calls) + self.assertIn("func3", data_lines[0]) + + # Test sort by time + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + self.mock_stats, sort=1, sample_interval_usec=100 + ) + + result = output.getvalue() + lines = result.strip().split("\n") + + data_lines = [l for l in lines if "file" in l and ".py" in l] + # func3 should be first (1.5s time) + self.assertIn("func3", data_lines[0]) + + def test_print_sampled_stats_limit(self): + """Test limiting output rows.""" + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + self.mock_stats, limit=2, sample_interval_usec=100 + ) + + result = output.getvalue() + + # Count function entries in the main stats section (not in summary) + lines = result.split("\n") + # Find where the main stats section ends (before summary) + main_section_lines = [] + for line in lines: + if "Summary of Interesting Functions:" in line: + break + main_section_lines.append(line) + + # Count function entries only in main section + func_count = sum( + 1 + for line in main_section_lines + if "func" in line and ".py" in line + ) + self.assertEqual(func_count, 2) + + def test_print_sampled_stats_time_units(self): + """Test proper time unit selection.""" + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats(self.mock_stats, sample_interval_usec=100) + + result = output.getvalue() + + # Should use seconds for the header since max time is > 1s + self.assertIn("tottime (s)", result) + self.assertIn("cumtime (s)", result) + + # Test with only microsecond-range times + micro_stats = mock.MagicMock() + micro_stats.stats = { + ("file1.py", 10, "func1"): (100, 100, 0.000005, 0.000010, {}), + } + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats(micro_stats, sample_interval_usec=100) + + result = output.getvalue() + + # Should use microseconds + self.assertIn("tottime (μs)", result) + self.assertIn("cumtime (μs)", result) + + def test_print_sampled_stats_summary(self): + """Test summary section generation.""" + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + self.mock_stats, + show_summary=True, + sample_interval_usec=100, + ) + + result = output.getvalue() + + # Check summary sections are present + self.assertIn("Summary of Interesting Functions:", result) + self.assertIn( + "Functions with Highest Direct/Cumulative Ratio (Hot Spots):", + result, + ) + self.assertIn( + "Functions with Highest Call Frequency (Indirect Calls):", result + ) + self.assertIn( + "Functions with Highest Call Magnification (Cumulative/Direct):", + result, + ) + + def test_print_sampled_stats_no_summary(self): + """Test disabling summary output.""" + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + self.mock_stats, + show_summary=False, + sample_interval_usec=100, + ) + + result = output.getvalue() + + # Summary should not be present + self.assertNotIn("Summary of Interesting Functions:", result) + + def test_print_sampled_stats_empty_stats(self): + """Test with empty stats.""" + + empty_stats = mock.MagicMock() + empty_stats.stats = {} + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats(empty_stats, sample_interval_usec=100) + + result = output.getvalue() + + # Should still print header + self.assertIn("Profile Stats:", result) + + def test_print_sampled_stats_sample_percentage_sorting(self): + """Test sample percentage sorting options.""" + + # Add a function with high sample percentage (more direct calls than func3's 200) + self.mock_stats.stats[("expensive.py", 60, "expensive_func")] = ( + 300, # direct calls (higher than func3's 200) + 300, # cumulative calls + 1.0, # total time + 1.0, # cumulative time + {}, + ) + + # Test sort by sample percentage + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + self.mock_stats, sort=3, sample_interval_usec=100 + ) # sample percentage + + result = output.getvalue() + lines = result.strip().split("\n") + + data_lines = [l for l in lines if ".py" in l and "func" in l] + # expensive_func should be first (highest sample percentage) + self.assertIn("expensive_func", data_lines[0]) + + def test_print_sampled_stats_with_recursive_calls(self): + """Test print_sampled_stats with recursive calls where nc != cc.""" + + # Create stats with recursive calls (nc != cc) + recursive_stats = mock.MagicMock() + recursive_stats.stats = { + # (direct_calls, cumulative_calls, tt, ct, callers) - recursive function + ("recursive.py", 10, "factorial"): ( + 5, # direct_calls + 10, # cumulative_calls (appears more times in stack due to recursion) + 0.5, + 0.6, + {}, + ), + ("normal.py", 20, "normal_func"): ( + 3, # direct_calls + 3, # cumulative_calls (same as direct for non-recursive) + 0.2, + 0.2, + {}, + ), + } + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats(recursive_stats, sample_interval_usec=100) + + result = output.getvalue() + + # Should display recursive calls as "5/10" format + self.assertIn("5/10", result) # nc/cc format for recursive calls + self.assertIn("3", result) # just nc for non-recursive calls + self.assertIn("factorial", result) + self.assertIn("normal_func", result) + + def test_print_sampled_stats_with_zero_call_counts(self): + """Test print_sampled_stats with zero call counts to trigger division protection.""" + + # Create stats with zero call counts + zero_stats = mock.MagicMock() + zero_stats.stats = { + ("file.py", 10, "zero_calls"): (0, 0, 0.0, 0.0, {}), # Zero calls + ("file.py", 20, "normal_func"): ( + 5, + 5, + 0.1, + 0.1, + {}, + ), # Normal function + } + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats(zero_stats, sample_interval_usec=100) + + result = output.getvalue() + + # Should handle zero call counts gracefully + self.assertIn("zero_calls", result) + self.assertIn("zero_calls", result) + self.assertIn("normal_func", result) + + def test_print_sampled_stats_sort_by_name(self): + """Test sort by function name option.""" + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + self.mock_stats, sort=-1, sample_interval_usec=100 + ) # sort by name + + result = output.getvalue() + lines = result.strip().split("\n") + + # Find the data lines (skip header and summary) + # Data lines start with whitespace and numbers, and contain filename:lineno(function) + data_lines = [] + for line in lines: + # Skip header lines and summary sections + if ( + line.startswith(" ") + and "(" in line + and ")" in line + and not line.startswith( + " 1." + ) # Skip summary lines that start with times + and not line.startswith( + " 0." + ) # Skip summary lines that start with times + and not "per call" in line # Skip summary lines + and not "calls" in line # Skip summary lines + and not "total time" in line # Skip summary lines + and not "cumulative time" in line + ): # Skip summary lines + data_lines.append(line) + + # Extract just the function names for comparison + func_names = [] + import re + + for line in data_lines: + # Function name is between the last ( and ), accounting for ANSI color codes + match = re.search(r"\(([^)]+)\)$", line) + if match: + func_name = match.group(1) + # Remove ANSI color codes + func_name = re.sub(r"\x1b\[[0-9;]*m", "", func_name) + func_names.append(func_name) + + # Verify we extracted function names and they are sorted + self.assertGreater( + len(func_names), 0, "Should have extracted some function names" + ) + self.assertEqual( + func_names, + sorted(func_names), + f"Function names {func_names} should be sorted alphabetically", + ) + + def test_print_sampled_stats_with_zero_time_functions(self): + """Test summary sections with functions that have zero time.""" + + # Create stats with zero-time functions + zero_time_stats = mock.MagicMock() + zero_time_stats.stats = { + ("file1.py", 10, "zero_time_func"): ( + 5, + 5, + 0.0, + 0.0, + {}, + ), # Zero time + ("file2.py", 20, "normal_func"): ( + 3, + 3, + 0.1, + 0.1, + {}, + ), # Normal time + } + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + zero_time_stats, + show_summary=True, + sample_interval_usec=100, + ) + + result = output.getvalue() + + # Should handle zero-time functions gracefully in summary + self.assertIn("Summary of Interesting Functions:", result) + self.assertIn("zero_time_func", result) + self.assertIn("normal_func", result) + + def test_print_sampled_stats_with_malformed_qualified_names(self): + """Test summary generation with function names that don't contain colons.""" + + # Create stats with function names that would create malformed qualified names + malformed_stats = mock.MagicMock() + malformed_stats.stats = { + # Function name without clear module separation + ("no_colon_func", 10, "func"): (3, 3, 0.1, 0.1, {}), + ("", 20, "empty_filename_func"): (2, 2, 0.05, 0.05, {}), + ("normal.py", 30, "normal_func"): (5, 5, 0.2, 0.2, {}), + } + + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + print_sampled_stats( + malformed_stats, + show_summary=True, + sample_interval_usec=100, + ) + + result = output.getvalue() + + # Should handle malformed names gracefully in summary aggregation + self.assertIn("Summary of Interesting Functions:", result) + # All function names should appear somewhere in the output + self.assertIn("func", result) + self.assertIn("empty_filename_func", result) + self.assertIn("normal_func", result) + + def test_print_sampled_stats_with_recursive_call_stats_creation(self): + """Test create_stats with recursive call data to trigger total_rec_calls branch.""" + collector = PstatsCollector(sample_interval_usec=1000000) # 1 second + + # Simulate recursive function data where total_rec_calls would be set + # We need to manually manipulate the collector result to test this branch + collector.result = { + ("recursive.py", 10, "factorial"): { + "total_rec_calls": 3, # Non-zero recursive calls + "direct_calls": 5, + "cumulative_calls": 10, + }, + ("normal.py", 20, "normal_func"): { + "total_rec_calls": 0, # Zero recursive calls + "direct_calls": 2, + "cumulative_calls": 5, + }, + } + + collector.create_stats() + + # Check that recursive calls are handled differently from non-recursive + factorial_stats = collector.stats[("recursive.py", 10, "factorial")] + normal_stats = collector.stats[("normal.py", 20, "normal_func")] + + # factorial should use cumulative_calls (10) as nc + self.assertEqual( + factorial_stats[1], 10 + ) # nc should be cumulative_calls + self.assertEqual(factorial_stats[0], 5) # cc should be direct_calls + + # normal_func should use cumulative_calls as nc + self.assertEqual(normal_stats[1], 5) # nc should be cumulative_calls + self.assertEqual(normal_stats[0], 2) # cc should be direct_calls diff --git a/Makefile.pre.in b/Makefile.pre.in index dd28ff5d2a3ed1..59c3c808794cf3 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2692,6 +2692,7 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_pathlib/support \ test/test_peg_generator \ test/test_profiling \ + test/test_profiling/test_sampling_profiler \ test/test_pydoc \ test/test_pyrepl \ test/test_string \ From 4695ec109d07c9bfd9eb7d91d6285c974a4331a7 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 18 Nov 2025 16:33:52 +0100 Subject: [PATCH 092/638] gh-138189: Link references to type slots (GH-141410) Link references to type slots --- Doc/c-api/structures.rst | 8 +++----- Doc/c-api/type.rst | 16 ++++++++-------- Doc/c-api/typeobj.rst | 2 +- Doc/howto/isolating-extensions.rst | 2 +- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 414dfdc84e61c9..b4e7cb1d77e1a3 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -698,14 +698,12 @@ The following flags can be used with :c:member:`PyMemberDef.flags`: entry indicates an offset from the subclass-specific data, rather than from ``PyObject``. - Can only be used as part of :c:member:`Py_tp_members ` + Can only be used as part of the :c:data:`Py_tp_members` :c:type:`slot ` when creating a class using negative :c:member:`~PyType_Spec.basicsize`. It is mandatory in that case. - - This flag is only used in :c:type:`PyType_Slot`. - When setting :c:member:`~PyTypeObject.tp_members` during - class creation, Python clears it and sets + When setting :c:member:`~PyTypeObject.tp_members` from the slot during + class creation, Python clears the flag and sets :c:member:`PyMemberDef.offset` to the offset from the ``PyObject`` struct. .. index:: diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index b608f815160f76..c7946e3190f01b 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -383,8 +383,8 @@ The following functions and structs are used to create The *bases* argument can be used to specify base classes; it can either be only one class or a tuple of classes. - If *bases* is ``NULL``, the *Py_tp_bases* slot is used instead. - If that also is ``NULL``, the *Py_tp_base* slot is used instead. + If *bases* is ``NULL``, the :c:data:`Py_tp_bases` slot is used instead. + If that also is ``NULL``, the :c:data:`Py_tp_base` slot is used instead. If that also is ``NULL``, the new type derives from :class:`object`. The *module* argument can be used to record the module in which the new @@ -590,9 +590,9 @@ The following functions and structs are used to create :c:type:`PyAsyncMethods` with an added ``Py_`` prefix. For example, use: - * ``Py_tp_dealloc`` to set :c:member:`PyTypeObject.tp_dealloc` - * ``Py_nb_add`` to set :c:member:`PyNumberMethods.nb_add` - * ``Py_sq_length`` to set :c:member:`PySequenceMethods.sq_length` + * :c:data:`Py_tp_dealloc` to set :c:member:`PyTypeObject.tp_dealloc` + * :c:data:`Py_nb_add` to set :c:member:`PyNumberMethods.nb_add` + * :c:data:`Py_sq_length` to set :c:member:`PySequenceMethods.sq_length` An additional slot is supported that does not correspond to a :c:type:`!PyTypeObject` struct field: @@ -611,7 +611,7 @@ The following functions and structs are used to create If it is not possible to switch to a ``MANAGED`` flag (for example, for vectorcall or to support Python older than 3.12), specify the - offset in :c:member:`Py_tp_members `. + offset in :c:data:`Py_tp_members`. See :ref:`PyMemberDef documentation ` for details. @@ -639,7 +639,7 @@ The following functions and structs are used to create .. versionchanged:: 3.14 The field :c:member:`~PyTypeObject.tp_vectorcall` can now set - using ``Py_tp_vectorcall``. See the field's documentation + using :c:data:`Py_tp_vectorcall`. See the field's documentation for details. .. c:member:: void *pfunc @@ -649,7 +649,7 @@ The following functions and structs are used to create *pfunc* values may not be ``NULL``, except for the following slots: - * ``Py_tp_doc`` + * :c:data:`Py_tp_doc` * :c:data:`Py_tp_token` (for clarity, prefer :c:data:`Py_TP_USE_SPEC` rather than ``NULL``) diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 34d19acdf17868..49fe02d919df8b 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -2273,7 +2273,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) This field should be set to ``NULL`` and treated as read-only. Python will fill it in when the type is :c:func:`initialized `. - For dynamically created classes, the ``Py_tp_bases`` + For dynamically created classes, the :c:data:`Py_tp_bases` :c:type:`slot ` can be used instead of the *bases* argument of :c:func:`PyType_FromSpecWithBases`. The argument form is preferred. diff --git a/Doc/howto/isolating-extensions.rst b/Doc/howto/isolating-extensions.rst index 7da6dc8a39795e..6092c75f48fdef 100644 --- a/Doc/howto/isolating-extensions.rst +++ b/Doc/howto/isolating-extensions.rst @@ -353,7 +353,7 @@ garbage collection protocol. That is, heap types should: - Have the :c:macro:`Py_TPFLAGS_HAVE_GC` flag. -- Define a traverse function using ``Py_tp_traverse``, which +- Define a traverse function using :c:data:`Py_tp_traverse`, which visits the type (e.g. using ``Py_VISIT(Py_TYPE(self))``). Please refer to the documentation of From 600f3feb234219c9a9998e30ea653a2afb1f8116 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 18 Nov 2025 17:13:13 +0100 Subject: [PATCH 093/638] gh-141070: Add PyUnstable_Object_Dump() function (#141072) * Promote _PyObject_Dump() as a public function. * Keep _PyObject_Dump() alias to PyUnstable_Object_Dump() for backward compatibility. * Replace _PyObject_Dump() with PyUnstable_Object_Dump(). Co-authored-by: Peter Bierma Co-authored-by: Kumar Aditya Co-authored-by: Petr Viktorin --- Doc/c-api/object.rst | 29 +++++++++++ Doc/whatsnew/3.15.rst | 12 +++-- Include/cpython/object.h | 14 +++-- .../pycore_global_objects_fini_generated.h | 2 +- Lib/test/test_capi/test_object.py | 52 +++++++++++++++++++ ...-11-05-21-48-31.gh-issue-141070.mkrhjQ.rst | 2 + Modules/_testcapi/object.c | 25 +++++++++ Objects/object.c | 4 +- Objects/unicodeobject.c | 3 +- Python/gc.c | 2 +- Python/pythonrun.c | 8 +-- 11 files changed, 135 insertions(+), 18 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-11-05-21-48-31.gh-issue-141070.mkrhjQ.rst diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 96353266ac7300..76971c46c1696b 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -85,6 +85,35 @@ Object Protocol instead of the :func:`repr`. +.. c:function:: void PyUnstable_Object_Dump(PyObject *op) + + Dump an object *op* to ``stderr``. This should only be used for debugging. + + The output is intended to try dumping objects even after memory corruption: + + * Information is written starting with fields that are the least likely to + crash when accessed. + * This function can be called without an :term:`attached thread state`, but + it's not recommended to do so: it can cause deadlocks. + * An object that does not belong to the current interpreter may be dumped, + but this may also cause crashes or unintended behavior. + * Implement a heuristic to detect if the object memory has been freed. Don't + display the object contents in this case, only its memory address. + * The output format may change at any time. + + Example of output: + + .. code-block:: output + + object address : 0x7f80124702c0 + object refcount : 2 + object type : 0x9902e0 + object type name: str + object repr : 'abcdef' + + .. versionadded:: next + + .. c:function:: int PyObject_HasAttrWithError(PyObject *o, PyObject *attr_name) Returns ``1`` if *o* has the attribute *attr_name*, and ``0`` otherwise. diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 24cc7e2d7eb911..5a98297d3f8847 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -1084,19 +1084,23 @@ New features (Contributed by Victor Stinner in :gh:`129813`.) +* Add a new :c:func:`PyImport_CreateModuleFromInitfunc` C-API for creating + a module from a *spec* and *initfunc*. + (Contributed by Itamar Oren in :gh:`116146`.) + * Add :c:func:`PyTuple_FromArray` to create a :class:`tuple` from an array. (Contributed by Victor Stinner in :gh:`111489`.) +* Add :c:func:`PyUnstable_Object_Dump` to dump an object to ``stderr``. + It should only be used for debugging. + (Contributed by Victor Stinner in :gh:`141070`.) + * Add :c:func:`PyUnstable_ThreadState_SetStackProtection` and :c:func:`PyUnstable_ThreadState_ResetStackProtection` functions to set the stack protection base address and stack protection size of a Python thread state. (Contributed by Victor Stinner in :gh:`139653`.) -* Add a new :c:func:`PyImport_CreateModuleFromInitfunc` C-API for creating - a module from a *spec* and *initfunc*. - (Contributed by Itamar Oren in :gh:`116146`.) - Changed C APIs -------------- diff --git a/Include/cpython/object.h b/Include/cpython/object.h index d64298232e705c..130a105de42150 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -295,7 +295,10 @@ PyAPI_FUNC(PyObject *) PyType_GetDict(PyTypeObject *); PyAPI_FUNC(int) PyObject_Print(PyObject *, FILE *, int); PyAPI_FUNC(void) _Py_BreakPoint(void); -PyAPI_FUNC(void) _PyObject_Dump(PyObject *); +PyAPI_FUNC(void) PyUnstable_Object_Dump(PyObject *); + +// Alias for backward compatibility +#define _PyObject_Dump PyUnstable_Object_Dump PyAPI_FUNC(PyObject*) _PyObject_GetAttrId(PyObject *, _Py_Identifier *); @@ -387,10 +390,11 @@ PyAPI_FUNC(PyObject *) _PyObject_FunctionStr(PyObject *); process with a message on stderr if the given condition fails to hold, but compile away to nothing if NDEBUG is defined. - However, before aborting, Python will also try to call _PyObject_Dump() on - the given object. This may be of use when investigating bugs in which a - particular object is corrupt (e.g. buggy a tp_visit method in an extension - module breaking the garbage collector), to help locate the broken objects. + However, before aborting, Python will also try to call + PyUnstable_Object_Dump() on the given object. This may be of use when + investigating bugs in which a particular object is corrupt (e.g. buggy a + tp_visit method in an extension module breaking the garbage collector), to + help locate the broken objects. The WITH_MSG variant allows you to supply an additional message that Python will attempt to print to stderr, after the object dump. */ diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index ecef4364cc32df..c3968aff8f3b8d 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -13,7 +13,7 @@ static inline void _PyStaticObject_CheckRefcnt(PyObject *obj) { if (!_Py_IsImmortal(obj)) { fprintf(stderr, "Immortal Object has less refcnt than expected.\n"); - _PyObject_Dump(obj); + PyUnstable_Object_Dump(obj); } } #endif diff --git a/Lib/test/test_capi/test_object.py b/Lib/test/test_capi/test_object.py index d4056727d07fbf..c5040913e9e1f1 100644 --- a/Lib/test/test_capi/test_object.py +++ b/Lib/test/test_capi/test_object.py @@ -1,4 +1,5 @@ import enum +import os import sys import textwrap import unittest @@ -13,6 +14,9 @@ _testcapi = import_helper.import_module('_testcapi') _testinternalcapi = import_helper.import_module('_testinternalcapi') +NULL = None +STDERR_FD = 2 + class Constant(enum.IntEnum): Py_CONSTANT_NONE = 0 @@ -247,5 +251,53 @@ def func(x): func(object()) + def pyobject_dump(self, obj, release_gil=False): + pyobject_dump = _testcapi.pyobject_dump + + try: + old_stderr = os.dup(STDERR_FD) + except OSError as exc: + # os.dup(STDERR_FD) is not supported on WASI + self.skipTest(f"os.dup() failed with {exc!r}") + + filename = os_helper.TESTFN + try: + try: + with open(filename, "wb") as fp: + fd = fp.fileno() + os.dup2(fd, STDERR_FD) + pyobject_dump(obj, release_gil) + finally: + os.dup2(old_stderr, STDERR_FD) + os.close(old_stderr) + + with open(filename) as fp: + return fp.read().rstrip() + finally: + os_helper.unlink(filename) + + def test_pyobject_dump(self): + # test string object + str_obj = 'test string' + output = self.pyobject_dump(str_obj) + hex_regex = r'(0x)?[0-9a-fA-F]+' + regex = ( + fr"object address : {hex_regex}\n" + r"object refcount : [0-9]+\n" + fr"object type : {hex_regex}\n" + r"object type name: str\n" + r"object repr : 'test string'" + ) + self.assertRegex(output, regex) + + # release the GIL + output = self.pyobject_dump(str_obj, release_gil=True) + self.assertRegex(output, regex) + + # test NULL object + output = self.pyobject_dump(NULL) + self.assertRegex(output, r'') + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/C_API/2025-11-05-21-48-31.gh-issue-141070.mkrhjQ.rst b/Misc/NEWS.d/next/C_API/2025-11-05-21-48-31.gh-issue-141070.mkrhjQ.rst new file mode 100644 index 00000000000000..39cfcf73404ebf --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-11-05-21-48-31.gh-issue-141070.mkrhjQ.rst @@ -0,0 +1,2 @@ +Add :c:func:`PyUnstable_Object_Dump` to dump an object to ``stderr``. It should +only be used for debugging. Patch by Victor Stinner. diff --git a/Modules/_testcapi/object.c b/Modules/_testcapi/object.c index 798ef97c495aeb..4c9632c07a99f4 100644 --- a/Modules/_testcapi/object.c +++ b/Modules/_testcapi/object.c @@ -485,6 +485,30 @@ is_uniquely_referenced(PyObject *self, PyObject *op) } +static PyObject * +pyobject_dump(PyObject *self, PyObject *args) +{ + PyObject *op; + int release_gil = 0; + + if (!PyArg_ParseTuple(args, "O|i", &op, &release_gil)) { + return NULL; + } + NULLABLE(op); + + if (release_gil) { + Py_BEGIN_ALLOW_THREADS + PyUnstable_Object_Dump(op); + Py_END_ALLOW_THREADS + + } + else { + PyUnstable_Object_Dump(op); + } + Py_RETURN_NONE; +} + + static PyMethodDef test_methods[] = { {"call_pyobject_print", call_pyobject_print, METH_VARARGS}, {"pyobject_print_null", pyobject_print_null, METH_VARARGS}, @@ -511,6 +535,7 @@ static PyMethodDef test_methods[] = { {"test_py_is_funcs", test_py_is_funcs, METH_NOARGS}, {"clear_managed_dict", clear_managed_dict, METH_O, NULL}, {"is_uniquely_referenced", is_uniquely_referenced, METH_O}, + {"pyobject_dump", pyobject_dump, METH_VARARGS}, {NULL}, }; diff --git a/Objects/object.c b/Objects/object.c index 0540112d7d2acf..0a80c6edcf158c 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -713,7 +713,7 @@ _PyObject_IsFreed(PyObject *op) /* For debugging convenience. See Misc/gdbinit for some useful gdb hooks */ void -_PyObject_Dump(PyObject* op) +PyUnstable_Object_Dump(PyObject* op) { if (_PyObject_IsFreed(op)) { /* It seems like the object memory has been freed: @@ -3150,7 +3150,7 @@ _PyObject_AssertFailed(PyObject *obj, const char *expr, const char *msg, /* This might succeed or fail, but we're about to abort, so at least try to provide any extra info we can: */ - _PyObject_Dump(obj); + PyUnstable_Object_Dump(obj); fprintf(stderr, "\n"); fflush(stderr); diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 4e8c132327b7d0..7f9f75126a9e56 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -547,7 +547,8 @@ unicode_check_encoding_errors(const char *encoding, const char *errors) } /* Disable checks during Python finalization. For example, it allows to - call _PyObject_Dump() during finalization for debugging purpose. */ + * call PyUnstable_Object_Dump() during finalization for debugging purpose. + */ if (_PyInterpreterState_GetFinalizing(interp) != NULL) { return 0; } diff --git a/Python/gc.c b/Python/gc.c index 064f9406e0a17c..27364ecfdcd5c6 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -2237,7 +2237,7 @@ _PyGC_Fini(PyInterpreterState *interp) void _PyGC_Dump(PyGC_Head *g) { - _PyObject_Dump(FROM_GC(g)); + PyUnstable_Object_Dump(FROM_GC(g)); } diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 49ce0a97d4742f..272be504a68fa1 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1181,7 +1181,7 @@ _PyErr_Display(PyObject *file, PyObject *unused, PyObject *value, PyObject *tb) } if (print_exception_recursive(&ctx, value) < 0) { PyErr_Clear(); - _PyObject_Dump(value); + PyUnstable_Object_Dump(value); fprintf(stderr, "lost sys.stderr\n"); } Py_XDECREF(ctx.seen); @@ -1199,14 +1199,14 @@ PyErr_Display(PyObject *unused, PyObject *value, PyObject *tb) PyObject *file; if (PySys_GetOptionalAttr(&_Py_ID(stderr), &file) < 0) { PyObject *exc = PyErr_GetRaisedException(); - _PyObject_Dump(value); + PyUnstable_Object_Dump(value); fprintf(stderr, "lost sys.stderr\n"); - _PyObject_Dump(exc); + PyUnstable_Object_Dump(exc); Py_DECREF(exc); return; } if (file == NULL) { - _PyObject_Dump(value); + PyUnstable_Object_Dump(value); fprintf(stderr, "lost sys.stderr\n"); return; } From daafacf0053e9c329b0f96447258f628dd0bd6f1 Mon Sep 17 00:00:00 2001 From: Shamil Date: Tue, 18 Nov 2025 19:34:58 +0300 Subject: [PATCH 094/638] gh-42400: Fix buffer overflow in _Py_wrealpath() for very long paths (#141529) Co-authored-by: Victor Stinner --- .../Security/2025-11-13-22-31-56.gh-issue-42400.pqB5Kq.rst | 3 +++ Python/fileutils.c | 7 ++++--- 2 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2025-11-13-22-31-56.gh-issue-42400.pqB5Kq.rst diff --git a/Misc/NEWS.d/next/Security/2025-11-13-22-31-56.gh-issue-42400.pqB5Kq.rst b/Misc/NEWS.d/next/Security/2025-11-13-22-31-56.gh-issue-42400.pqB5Kq.rst new file mode 100644 index 00000000000000..17dc241aef91d6 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2025-11-13-22-31-56.gh-issue-42400.pqB5Kq.rst @@ -0,0 +1,3 @@ +Fix buffer overflow in ``_Py_wrealpath()`` for paths exceeding ``MAXPATHLEN`` bytes +by using dynamic memory allocation instead of fixed-size buffer. +Patch by Shamil Abdulaev. diff --git a/Python/fileutils.c b/Python/fileutils.c index 93abd70a34d420..0c1766b8804500 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2118,7 +2118,6 @@ _Py_wrealpath(const wchar_t *path, wchar_t *resolved_path, size_t resolved_path_len) { char *cpath; - char cresolved_path[MAXPATHLEN]; wchar_t *wresolved_path; char *res; size_t r; @@ -2127,12 +2126,14 @@ _Py_wrealpath(const wchar_t *path, errno = EINVAL; return NULL; } - res = realpath(cpath, cresolved_path); + res = realpath(cpath, NULL); PyMem_RawFree(cpath); if (res == NULL) return NULL; - wresolved_path = Py_DecodeLocale(cresolved_path, &r); + wresolved_path = Py_DecodeLocale(res, &r); + free(res); + if (wresolved_path == NULL) { errno = EINVAL; return NULL; From 4cfa695c953e5dfdab99ade81cee960ddf4b106d Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Tue, 18 Nov 2025 09:51:18 -0800 Subject: [PATCH 095/638] GH-141686: Break cycles created by JSONEncoder.iterencode (GH-141687) --- Lib/json/encoder.py | 30 +++++++++---------- ...-11-17-16-53-49.gh-issue-141686.V-xaoI.rst | 2 ++ 2 files changed, 16 insertions(+), 16 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-17-16-53-49.gh-issue-141686.V-xaoI.rst diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index 5cf6d64f3eade6..4c70e8b75ed132 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -264,17 +264,6 @@ def floatstr(o, allow_nan=self.allow_nan, def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot, - ## HACK: hand-optimized bytecode; turn globals into locals - ValueError=ValueError, - dict=dict, - float=float, - id=id, - int=int, - isinstance=isinstance, - list=list, - str=str, - tuple=tuple, - _intstr=int.__repr__, ): def _iterencode_list(lst, _current_indent_level): @@ -311,7 +300,7 @@ def _iterencode_list(lst, _current_indent_level): # Subclasses of int/float may override __repr__, but we still # want to encode them as integers/floats in JSON. One example # within the standard library is IntEnum. - yield buf + _intstr(value) + yield buf + int.__repr__(value) elif isinstance(value, float): # see comment above for int yield buf + _floatstr(value) @@ -374,7 +363,7 @@ def _iterencode_dict(dct, _current_indent_level): key = 'null' elif isinstance(key, int): # see comment for int/float in _make_iterencode - key = _intstr(key) + key = int.__repr__(key) elif _skipkeys: continue else: @@ -399,7 +388,7 @@ def _iterencode_dict(dct, _current_indent_level): yield 'false' elif isinstance(value, int): # see comment for int/float in _make_iterencode - yield _intstr(value) + yield int.__repr__(value) elif isinstance(value, float): # see comment for int/float in _make_iterencode yield _floatstr(value) @@ -434,7 +423,7 @@ def _iterencode(o, _current_indent_level): yield 'false' elif isinstance(o, int): # see comment for int/float in _make_iterencode - yield _intstr(o) + yield int.__repr__(o) elif isinstance(o, float): # see comment for int/float in _make_iterencode yield _floatstr(o) @@ -458,4 +447,13 @@ def _iterencode(o, _current_indent_level): raise if markers is not None: del markers[markerid] - return _iterencode + + def _iterencode_once(o, _current_indent_level): + nonlocal _iterencode, _iterencode_dict, _iterencode_list + try: + yield from _iterencode(o, _current_indent_level) + finally: + # Break reference cycles due to mutually recursive closures: + del _iterencode, _iterencode_dict, _iterencode_list + + return _iterencode_once diff --git a/Misc/NEWS.d/next/Library/2025-11-17-16-53-49.gh-issue-141686.V-xaoI.rst b/Misc/NEWS.d/next/Library/2025-11-17-16-53-49.gh-issue-141686.V-xaoI.rst new file mode 100644 index 00000000000000..87e9cb8d69bfbc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-17-16-53-49.gh-issue-141686.V-xaoI.rst @@ -0,0 +1,2 @@ +Break reference cycles created by each call to :func:`json.dump` or +:meth:`json.JSONEncoder.iterencode`. From 17636ba48ce535fc1b1926c0bab26339da50631a Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Wed, 19 Nov 2025 06:39:21 +0800 Subject: [PATCH 096/638] gh-141691: Apply ruff rules to Apple folder. (#141694) Add ruff configuration to run over the Apple build tooling and testbed script. --- .pre-commit-config.yaml | 8 ++ Apple/.ruff.toml | 22 ++++++ Apple/__main__.py | 154 +++++++++++++++++++------------------- Apple/testbed/__main__.py | 33 ++++---- 4 files changed, 126 insertions(+), 91 deletions(-) create mode 100644 Apple/.ruff.toml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b0311f052798ad..c5767ee841eb0d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,6 +2,10 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.13.2 hooks: + - id: ruff-check + name: Run Ruff (lint) on Apple/ + args: [--exit-non-zero-on-fix, --config=Apple/.ruff.toml] + files: ^Apple/ - id: ruff-check name: Run Ruff (lint) on Doc/ args: [--exit-non-zero-on-fix] @@ -30,6 +34,10 @@ repos: name: Run Ruff (lint) on Tools/wasm/ args: [--exit-non-zero-on-fix, --config=Tools/wasm/.ruff.toml] files: ^Tools/wasm/ + - id: ruff-format + name: Run Ruff (format) on Apple/ + args: [--exit-non-zero-on-fix, --config=Apple/.ruff.toml] + files: ^Apple - id: ruff-format name: Run Ruff (format) on Doc/ args: [--check] diff --git a/Apple/.ruff.toml b/Apple/.ruff.toml new file mode 100644 index 00000000000000..4cdc39ebee4be9 --- /dev/null +++ b/Apple/.ruff.toml @@ -0,0 +1,22 @@ +extend = "../.ruff.toml" # Inherit the project-wide settings + +[format] +preview = true +docstring-code-format = true + +[lint] +select = [ + "C4", # flake8-comprehensions + "E", # pycodestyle + "F", # pyflakes + "I", # isort + "ISC", # flake8-implicit-str-concat + "LOG", # flake8-logging + "PGH", # pygrep-hooks + "PT", # flake8-pytest-style + "PYI", # flake8-pyi + "RUF100", # Ban unused `# noqa` comments + "UP", # pyupgrade + "W", # pycodestyle + "YTT", # flake8-2020 +] diff --git a/Apple/__main__.py b/Apple/__main__.py index e76fc351798707..1c588c23d6b5d1 100644 --- a/Apple/__main__.py +++ b/Apple/__main__.py @@ -46,13 +46,12 @@ import sys import sysconfig import time -from collections.abc import Sequence +from collections.abc import Callable, Sequence from contextlib import contextmanager from datetime import datetime, timezone from os.path import basename, relpath from pathlib import Path from subprocess import CalledProcessError -from typing import Callable EnvironmentT = dict[str, str] ArgsT = Sequence[str | Path] @@ -140,17 +139,15 @@ def print_env(env: EnvironmentT) -> None: def apple_env(host: str) -> EnvironmentT: """Construct an Apple development environment for the given host.""" env = { - "PATH": ":".join( - [ - str(PYTHON_DIR / "Apple/iOS/Resources/bin"), - str(subdir(host) / "prefix"), - "/usr/bin", - "/bin", - "/usr/sbin", - "/sbin", - "/Library/Apple/usr/bin", - ] - ), + "PATH": ":".join([ + str(PYTHON_DIR / "Apple/iOS/Resources/bin"), + str(subdir(host) / "prefix"), + "/usr/bin", + "/bin", + "/usr/sbin", + "/sbin", + "/Library/Apple/usr/bin", + ]), } return env @@ -196,14 +193,10 @@ def clean(context: argparse.Namespace, target: str = "all") -> None: paths.append(target) if target in {"all", "hosts", "test"}: - paths.extend( - [ - path.name - for path in CROSS_BUILD_DIR.glob( - f"{context.platform}-testbed.*" - ) - ] - ) + paths.extend([ + path.name + for path in CROSS_BUILD_DIR.glob(f"{context.platform}-testbed.*") + ]) for path in paths: delete_path(path) @@ -352,18 +345,16 @@ def download(url: str, target_dir: Path) -> Path: out_path = target_path / basename(url) if not Path(out_path).is_file(): - run( - [ - "curl", - "-Lf", - "--retry", - "5", - "--retry-all-errors", - "-o", - out_path, - url, - ] - ) + run([ + "curl", + "-Lf", + "--retry", + "5", + "--retry-all-errors", + "-o", + out_path, + url, + ]) else: print(f"Using cached version of {basename(url)}") return out_path @@ -468,8 +459,7 @@ def package_version(prefix_path: Path) -> str: def lib_platform_files(dirname, names): - """A file filter that ignores platform-specific files in the lib directory. - """ + """A file filter that ignores platform-specific files in lib.""" path = Path(dirname) if ( path.parts[-3] == "lib" @@ -478,7 +468,7 @@ def lib_platform_files(dirname, names): ): return names elif path.parts[-2] == "lib" and path.parts[-1].startswith("python"): - ignored_names = set( + ignored_names = { name for name in names if ( @@ -486,7 +476,7 @@ def lib_platform_files(dirname, names): or name.startswith("_sysconfig_vars_") or name == "build-details.json" ) - ) + } else: ignored_names = set() @@ -499,7 +489,9 @@ def lib_non_platform_files(dirname, names): """ path = Path(dirname) if path.parts[-2] == "lib" and path.parts[-1].startswith("python"): - return set(names) - lib_platform_files(dirname, names) - {"lib-dynload"} + return ( + set(names) - lib_platform_files(dirname, names) - {"lib-dynload"} + ) else: return set() @@ -514,7 +506,8 @@ def create_xcframework(platform: str) -> str: package_path.mkdir() except FileExistsError: raise RuntimeError( - f"{platform} XCframework already exists; do you need to run with --clean?" + f"{platform} XCframework already exists; do you need to run " + "with --clean?" ) from None frameworks = [] @@ -607,7 +600,7 @@ def create_xcframework(platform: str) -> str: print(f" - {slice_name} binaries") shutil.copytree(first_path / "bin", slice_path / "bin") - # Copy the include path (this will be a symlink to the framework headers) + # Copy the include path (a symlink to the framework headers) print(f" - {slice_name} include files") shutil.copytree( first_path / "include", @@ -659,7 +652,8 @@ def create_xcframework(platform: str) -> str: # statically link those libraries into a Framework, you become # responsible for providing a privacy manifest for that framework. xcprivacy_file = { - "OpenSSL": subdir(host_triple) / "prefix/share/OpenSSL.xcprivacy" + "OpenSSL": subdir(host_triple) + / "prefix/share/OpenSSL.xcprivacy" } print(f" - {multiarch} xcprivacy files") for module, lib in [ @@ -669,7 +663,8 @@ def create_xcframework(platform: str) -> str: shutil.copy( xcprivacy_file[lib], slice_path - / f"lib-{arch}/python{version_tag}/lib-dynload/{module}.xcprivacy", + / f"lib-{arch}/python{version_tag}" + / f"lib-dynload/{module}.xcprivacy", ) print(" - build tools") @@ -692,18 +687,16 @@ def package(context: argparse.Namespace) -> None: # Clone testbed print() - run( - [ - sys.executable, - "Apple/testbed", - "clone", - "--platform", - context.platform, - "--framework", - CROSS_BUILD_DIR / context.platform / "Python.xcframework", - CROSS_BUILD_DIR / context.platform / "testbed", - ] - ) + run([ + sys.executable, + "Apple/testbed", + "clone", + "--platform", + context.platform, + "--framework", + CROSS_BUILD_DIR / context.platform / "Python.xcframework", + CROSS_BUILD_DIR / context.platform / "testbed", + ]) # Build the final archive archive_name = ( @@ -757,7 +750,7 @@ def build(context: argparse.Namespace, host: str | None = None) -> None: package(context) -def test(context: argparse.Namespace, host: str | None = None) -> None: +def test(context: argparse.Namespace, host: str | None = None) -> None: # noqa: PT028 """The implementation of the "test" command.""" if host is None: host = context.host @@ -795,18 +788,16 @@ def test(context: argparse.Namespace, host: str | None = None) -> None: / f"Frameworks/{apple_multiarch(host)}" ) - run( - [ - sys.executable, - "Apple/testbed", - "clone", - "--platform", - context.platform, - "--framework", - framework_path, - testbed_dir, - ] - ) + run([ + sys.executable, + "Apple/testbed", + "clone", + "--platform", + context.platform, + "--framework", + framework_path, + testbed_dir, + ]) run( [ @@ -840,7 +831,7 @@ def apple_sim_host(platform_name: str) -> str: """Determine the native simulator target for this platform.""" for _, slice_parts in HOSTS[platform_name].items(): for host_triple in slice_parts: - parts = host_triple.split('-') + parts = host_triple.split("-") if parts[0] == platform.machine() and parts[-1] == "simulator": return host_triple @@ -968,20 +959,29 @@ def parse_args() -> argparse.Namespace: cmd.add_argument( "--simulator", help=( - "The name of the simulator to use (eg: 'iPhone 16e'). Defaults to " - "the most recently released 'entry level' iPhone device. Device " - "architecture and OS version can also be specified; e.g., " - "`--simulator 'iPhone 16 Pro,arch=arm64,OS=26.0'` would run on " - "an ARM64 iPhone 16 Pro simulator running iOS 26.0." + "The name of the simulator to use (eg: 'iPhone 16e'). " + "Defaults to the most recently released 'entry level' " + "iPhone device. Device architecture and OS version can also " + "be specified; e.g., " + "`--simulator 'iPhone 16 Pro,arch=arm64,OS=26.0'` would " + "run on an ARM64 iPhone 16 Pro simulator running iOS 26.0." ), ) group = cmd.add_mutually_exclusive_group() group.add_argument( - "--fast-ci", action="store_const", dest="ci_mode", const="fast", - help="Add test arguments for GitHub Actions") + "--fast-ci", + action="store_const", + dest="ci_mode", + const="fast", + help="Add test arguments for GitHub Actions", + ) group.add_argument( - "--slow-ci", action="store_const", dest="ci_mode", const="slow", - help="Add test arguments for buildbots") + "--slow-ci", + action="store_const", + dest="ci_mode", + const="slow", + help="Add test arguments for buildbots", + ) for subcommand in [configure_build, configure_host, build, ci]: subcommand.add_argument( diff --git a/Apple/testbed/__main__.py b/Apple/testbed/__main__.py index 49974cb142853c..0dd77ab8b82797 100644 --- a/Apple/testbed/__main__.py +++ b/Apple/testbed/__main__.py @@ -32,15 +32,15 @@ def select_simulator_device(platform): json_data = json.loads(raw_json) if platform == "iOS": - # Any iOS device will do; we'll look for "SE" devices - but the name isn't - # consistent over time. Older Xcode versions will use "iPhone SE (Nth - # generation)"; As of 2025, they've started using "iPhone 16e". + # Any iOS device will do; we'll look for "SE" devices - but the name + # isn't consistent over time. Older Xcode versions will use "iPhone SE + # (Nth generation)"; As of 2025, they've started using "iPhone 16e". # - # When Xcode is updated after a new release, new devices will be available - # and old ones will be dropped from the set available on the latest iOS - # version. Select the one with the highest minimum runtime version - this - # is an indicator of the "newest" released device, which should always be - # supported on the "most recent" iOS version. + # When Xcode is updated after a new release, new devices will be + # available and old ones will be dropped from the set available on the + # latest iOS version. Select the one with the highest minimum runtime + # version - this is an indicator of the "newest" released device, which + # should always be supported on the "most recent" iOS version. se_simulators = sorted( (devicetype["minRuntimeVersion"], devicetype["name"]) for devicetype in json_data["devicetypes"] @@ -295,7 +295,8 @@ def main(): parser = argparse.ArgumentParser( description=( - "Manages the process of testing an Apple Python project through Xcode." + "Manages the process of testing an Apple Python project " + "through Xcode." ), ) @@ -336,7 +337,10 @@ def main(): run = subcommands.add_parser( "run", - usage="%(prog)s [-h] [--simulator SIMULATOR] -- [ ...]", + usage=( + "%(prog)s [-h] [--simulator SIMULATOR] -- " + " [ ...]" + ), description=( "Run a testbed project. The arguments provided after `--` will be " "passed to the running iOS process as if they were arguments to " @@ -397,9 +401,9 @@ def main(): / "bin" ).is_dir(): print( - f"Testbed does not contain a compiled Python framework. Use " - f"`python {sys.argv[0]} clone ...` to create a runnable " - f"clone of this testbed." + "Testbed does not contain a compiled Python framework. " + f"Use `python {sys.argv[0]} clone ...` to create a " + "runnable clone of this testbed." ) sys.exit(20) @@ -411,7 +415,8 @@ def main(): ) else: print( - f"Must specify test arguments (e.g., {sys.argv[0]} run -- test)" + "Must specify test arguments " + f"(e.g., {sys.argv[0]} run -- test)" ) print() parser.print_help(sys.stderr) From 652c764a59913327b28b32016405696a620d969e Mon Sep 17 00:00:00 2001 From: Thierry Martos <81799048+ThierryMT@users.noreply.github.com> Date: Tue, 18 Nov 2025 16:01:09 -0800 Subject: [PATCH 097/638] gh-140381: Increase slow_fibonacci call frequency in test_profiling (#140673) --- .../test_profiling/test_sampling_profiler/test_integration.py | 4 ++-- .../next/Tests/2025-10-27-15-53-47.gh-issue-140381.N5o3pa.rst | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2025-10-27-15-53-47.gh-issue-140381.N5o3pa.rst diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_integration.py b/Lib/test/test_profiling/test_sampling_profiler/test_integration.py index 4fb2c595bbef9a..e1c80fa6d5d1b7 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_integration.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_integration.py @@ -414,8 +414,8 @@ def main_loop(): if iteration % 3 == 0: # Very CPU intensive result = cpu_intensive_work() - elif iteration % 5 == 0: - # Expensive recursive operation + elif iteration % 2 == 0: + # Expensive recursive operation (increased frequency for slower machines) result = slow_fibonacci(12) else: # Medium operation diff --git a/Misc/NEWS.d/next/Tests/2025-10-27-15-53-47.gh-issue-140381.N5o3pa.rst b/Misc/NEWS.d/next/Tests/2025-10-27-15-53-47.gh-issue-140381.N5o3pa.rst new file mode 100644 index 00000000000000..568a2b65d7d204 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2025-10-27-15-53-47.gh-issue-140381.N5o3pa.rst @@ -0,0 +1 @@ +Fix flaky test_profiling tests on i686 and s390x architectures by increasing slow_fibonacci call frequency from every 5th iteration to every 2nd iteration. From ce791541769a41beabec0f515cd62e504d46ff1c Mon Sep 17 00:00:00 2001 From: Edward Xu Date: Wed, 19 Nov 2025 08:57:59 +0800 Subject: [PATCH 098/638] gh-139103: fix free-threading `dataclass.__init__` perf issue (gh-141596) The dataclasses `__init__` function is generated dynamically by a call to `exec()` and so doesn't have deferred reference counting enabled. Enable deferred reference counting on functions when assigned as an attribute to type objects to avoid reference count contention when creating dataclass instances. --- .../2025-11-15-23-58-23.gh-issue-139103.9cVYJ0.rst | 1 + Objects/typeobject.c | 12 ++++++++++++ Tools/ftscalingbench/ftscalingbench.py | 12 ++++++++++++ 3 files changed, 25 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-23-58-23.gh-issue-139103.9cVYJ0.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-23-58-23.gh-issue-139103.9cVYJ0.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-23-58-23.gh-issue-139103.9cVYJ0.rst new file mode 100644 index 00000000000000..c038dc742ccec9 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-15-23-58-23.gh-issue-139103.9cVYJ0.rst @@ -0,0 +1 @@ +Improve multithreaded scaling of dataclasses on the free-threaded build. diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 61bcc21ce13d47..c99c6b3f6377b6 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -6546,6 +6546,18 @@ type_setattro(PyObject *self, PyObject *name, PyObject *value) assert(!_PyType_HasFeature(metatype, Py_TPFLAGS_INLINE_VALUES)); assert(!_PyType_HasFeature(metatype, Py_TPFLAGS_MANAGED_DICT)); +#ifdef Py_GIL_DISABLED + // gh-139103: Enable deferred refcounting for functions assigned + // to type objects. This is important for `dataclass.__init__`, + // which is generated dynamically. + if (value != NULL && + PyFunction_Check(value) && + !_PyObject_HasDeferredRefcount(value)) + { + PyUnstable_Object_EnableDeferredRefcount(value); + } +#endif + PyObject *old_value = NULL; PyObject *descr = _PyType_LookupRef(metatype, name); if (descr != NULL) { diff --git a/Tools/ftscalingbench/ftscalingbench.py b/Tools/ftscalingbench/ftscalingbench.py index 1a59e25189d5dd..097a065f368f30 100644 --- a/Tools/ftscalingbench/ftscalingbench.py +++ b/Tools/ftscalingbench/ftscalingbench.py @@ -27,6 +27,7 @@ import sys import threading import time +from dataclasses import dataclass from operator import methodcaller # The iterations in individual benchmarks are scaled by this factor. @@ -202,6 +203,17 @@ def method_caller(): for i in range(1000 * WORK_SCALE): mc(obj) +@dataclass +class MyDataClass: + x: int + y: int + z: int + +@register_benchmark +def instantiate_dataclass(): + for _ in range(1000 * WORK_SCALE): + obj = MyDataClass(x=1, y=2, z=3) + def bench_one_thread(func): t0 = time.perf_counter_ns() func() From 7b0b70867586ef7109de60ccce94d13164dbb776 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Wed, 19 Nov 2025 09:48:51 +0800 Subject: [PATCH 099/638] gh-141692: Add a slice-specific lib folder to iOS XCframeworks. (#141693) Modifies the iOS XCframework to include a lib folder for each slice that contains a symlinked version of the libPython dynamic library. --- Apple/__main__.py | 14 ++++++++++++++ Apple/testbed/Python.xcframework/build/utils.sh | 3 ++- Makefile.pre.in | 3 +++ .../2025-11-18-13-55-47.gh-issue-141692.tud9if.rst | 3 +++ 4 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2025-11-18-13-55-47.gh-issue-141692.tud9if.rst diff --git a/Apple/__main__.py b/Apple/__main__.py index 1c588c23d6b5d1..256966e76c2c97 100644 --- a/Apple/__main__.py +++ b/Apple/__main__.py @@ -477,6 +477,12 @@ def lib_platform_files(dirname, names): or name == "build-details.json" ) } + elif path.parts[-1] == "lib": + ignored_names = { + name + for name in names + if name.startswith("libpython") and name.endswith(".dylib") + } else: ignored_names = set() @@ -614,6 +620,12 @@ def create_xcframework(platform: str) -> str: slice_framework / "Headers/pyconfig.h", ) + print(f" - {slice_name} shared library") + # Create a simlink for the fat library + shared_lib = slice_path / f"lib/libpython{version_tag}.dylib" + shared_lib.parent.mkdir() + shared_lib.symlink_to("../Python.framework/Python") + print(f" - {slice_name} architecture-specific files") for host_triple, multiarch in slice_parts.items(): print(f" - {multiarch} standard library") @@ -625,6 +637,7 @@ def create_xcframework(platform: str) -> str: framework_path(host_triple, multiarch) / "lib", package_path / "Python.xcframework/lib", ignore=lib_platform_files, + symlinks=True, ) has_common_stdlib = True @@ -632,6 +645,7 @@ def create_xcframework(platform: str) -> str: framework_path(host_triple, multiarch) / "lib", slice_path / f"lib-{arch}", ignore=lib_non_platform_files, + symlinks=True, ) # Copy the host's pyconfig.h to an architecture-specific name. diff --git a/Apple/testbed/Python.xcframework/build/utils.sh b/Apple/testbed/Python.xcframework/build/utils.sh index 961c46d014b5f5..e7155d8b30e213 100755 --- a/Apple/testbed/Python.xcframework/build/utils.sh +++ b/Apple/testbed/Python.xcframework/build/utils.sh @@ -46,7 +46,8 @@ install_stdlib() { rsync -au --delete "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/lib/" "$CODESIGNING_FOLDER_PATH/python/lib/" rsync -au "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/$SLICE_FOLDER/lib-$ARCHS/" "$CODESIGNING_FOLDER_PATH/python/lib/" else - rsync -au --delete "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/$SLICE_FOLDER/lib/" "$CODESIGNING_FOLDER_PATH/python/lib/" + # A single-arch framework will have a libpython symlink; that can't be included at runtime + rsync -au --delete "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/$SLICE_FOLDER/lib/" "$CODESIGNING_FOLDER_PATH/python/lib/" --exclude 'libpython*.dylib' fi } diff --git a/Makefile.pre.in b/Makefile.pre.in index 59c3c808794cf3..13108b1baf976a 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -3050,6 +3050,9 @@ frameworkinstallunversionedstructure: $(LDLIBRARY) $(INSTALL) -d -m $(DIRMODE) $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR) sed 's/%VERSION%/'"`$(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import platform; print(platform.python_version())'`"'/g' < $(RESSRCDIR)/Info.plist > $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Info.plist $(INSTALL_SHARED) $(LDLIBRARY) $(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/$(LDLIBRARY) + $(INSTALL) -d -m $(DIRMODE) $(DESTDIR)$(LIBDIR) + $(LN) -fs "../$(LDLIBRARY)" "$(DESTDIR)$(prefix)/lib/libpython$(LDVERSION).dylib" + $(LN) -fs "../$(LDLIBRARY)" "$(DESTDIR)$(prefix)/lib/libpython$(VERSION).dylib" $(INSTALL) -d -m $(DIRMODE) $(DESTDIR)$(BINDIR) for file in $(srcdir)/$(RESSRCDIR)/bin/* ; do \ $(INSTALL) -m $(EXEMODE) $$file $(DESTDIR)$(BINDIR); \ diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-11-18-13-55-47.gh-issue-141692.tud9if.rst b/Misc/NEWS.d/next/Tools-Demos/2025-11-18-13-55-47.gh-issue-141692.tud9if.rst new file mode 100644 index 00000000000000..d85c54db3646f6 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2025-11-18-13-55-47.gh-issue-141692.tud9if.rst @@ -0,0 +1,3 @@ +Each slice of an iOS XCframework now contains a ``lib`` folder that contains +a symlink to the libpython dylib. This allows binary modules to be compiled +for iOS using dynamic libreary linking, rather than Framework linking. From 92c5de73b8d7526326c865b1a669b868f0d40c1e Mon Sep 17 00:00:00 2001 From: Ayappan Perumal Date: Wed, 19 Nov 2025 13:07:09 +0530 Subject: [PATCH 100/638] gh-141659: Fix bad file descriptor error in subprocess on AIX (GH-141660) /proc/self does not exist on AIX. --- .../2025-11-17-08-16-30.gh-issue-141659.QNi9Aj.rst | 1 + Modules/_posixsubprocess.c | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-17-08-16-30.gh-issue-141659.QNi9Aj.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-17-08-16-30.gh-issue-141659.QNi9Aj.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-17-08-16-30.gh-issue-141659.QNi9Aj.rst new file mode 100644 index 00000000000000..eeb055c6012a12 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-17-08-16-30.gh-issue-141659.QNi9Aj.rst @@ -0,0 +1 @@ +Fix bad file descriptor errors from ``_posixsubprocess`` on AIX. diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index 0219a3360fd6b1..6f0a6d1d4e37fe 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -514,7 +514,13 @@ _close_open_fds_maybe_unsafe(int start_fd, int *fds_to_keep, proc_fd_dir = NULL; else #endif +#if defined(_AIX) + char fd_path[PATH_MAX]; + snprintf(fd_path, sizeof(fd_path), "/proc/%ld/fd", (long)getpid()); + proc_fd_dir = opendir(fd_path); +#else proc_fd_dir = opendir(FD_DIR); +#endif if (!proc_fd_dir) { /* No way to get a list of open fds. */ _close_range_except(start_fd, -1, fds_to_keep, fds_to_keep_len, From dbbf4b2e21d4aa73f54361ecda12187bacd7f6d3 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Wed, 19 Nov 2025 11:42:16 +0200 Subject: [PATCH 101/638] Post 3.15.0a2 --- Include/patchlevel.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 899c892631fafa..804aa1a0427ba9 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -27,7 +27,7 @@ #define PY_RELEASE_SERIAL 2 /* Version as a string */ -#define PY_VERSION "3.15.0a2" +#define PY_VERSION "3.15.0a2+" /*--end constants--*/ From c25a070759952b13f97ecc37ca2991c2669aee47 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 19 Nov 2025 10:16:24 +0000 Subject: [PATCH 102/638] GH-139653: Only raise an exception (or fatal error) when the stack pointer is about to overflow the stack. (GH-141711) Only raises if the stack pointer is both below the limit *and* above the stack base. This prevents false positives for user-space threads, as the stack pointer will be outside those bounds if the stack has been swapped. --- Include/internal/pycore_ceval.h | 7 +++-- InternalDocs/stack_protection.md | 9 +++++- ...-11-17-14-40-45.gh-issue-139653.LzOy1M.rst | 4 +++ Python/ceval.c | 28 ++++++++++++++----- 4 files changed, 38 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-17-14-40-45.gh-issue-139653.LzOy1M.rst diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 47c42fccdc2376..2ae84be7b33966 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -217,10 +217,13 @@ extern void _PyEval_DeactivateOpCache(void); static inline int _Py_MakeRecCheck(PyThreadState *tstate) { uintptr_t here_addr = _Py_get_machine_stack_pointer(); _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + // Overflow if stack pointer is between soft limit and the base of the hardware stack. + // If it is below the hardware stack base, assume that we have the wrong stack limits, and do nothing. + // We could have the wrong stack limits because of limited platform support, or user-space threads. #if _Py_STACK_GROWS_DOWN - return here_addr < _tstate->c_stack_soft_limit; + return here_addr < _tstate->c_stack_soft_limit && here_addr >= _tstate->c_stack_soft_limit - 2 * _PyOS_STACK_MARGIN_BYTES; #else - return here_addr > _tstate->c_stack_soft_limit; + return here_addr > _tstate->c_stack_soft_limit && here_addr <= _tstate->c_stack_soft_limit + 2 * _PyOS_STACK_MARGIN_BYTES; #endif } diff --git a/InternalDocs/stack_protection.md b/InternalDocs/stack_protection.md index fa025bd930f74e..14802e57d095f4 100644 --- a/InternalDocs/stack_protection.md +++ b/InternalDocs/stack_protection.md @@ -38,12 +38,19 @@ Recursion checks are performed by `_Py_EnterRecursiveCall()` or `_Py_EnterRecurs ```python kb_used = (stack_top - stack_pointer)>>10 -if stack_pointer < hard_limit: +if stack_pointer < bottom_of_machine_stack: + pass # Our stack limits could be wrong so it is safest to do nothing. +elif stack_pointer < hard_limit: FatalError(f"Unrecoverable stack overflow (used {kb_used} kB)") elif stack_pointer < soft_limit: raise RecursionError(f"Stack overflow (used {kb_used} kB)") ``` +### User space threads and other oddities + +Some libraries provide user-space threads. These will change the C stack at runtime. +To guard against this we only raise if the stack pointer is in the window between the expected stack base and the soft limit. + ### Diagnosing and fixing stack overflows For stack protection to work correctly the amount of stack consumed between calls to `_Py_EnterRecursiveCall()` must be less than `_PyOS_STACK_MARGIN_BYTES`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-17-14-40-45.gh-issue-139653.LzOy1M.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-17-14-40-45.gh-issue-139653.LzOy1M.rst new file mode 100644 index 00000000000000..c3ae0e8adab319 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-17-14-40-45.gh-issue-139653.LzOy1M.rst @@ -0,0 +1,4 @@ +Only raise a ``RecursionError`` or trigger a fatal error if the stack +pointer is both below the limit pointer *and* above the stack base. If +outside of these bounds assume that it is OK. This prevents false positives +when user-space threads swap stacks. diff --git a/Python/ceval.c b/Python/ceval.c index 14fef42ea967d6..5381cd826dfd19 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -362,9 +362,11 @@ _Py_ReachedRecursionLimitWithMargin(PyThreadState *tstate, int margin_count) _Py_InitializeRecursionLimits(tstate); } #if _Py_STACK_GROWS_DOWN - return here_addr <= _tstate->c_stack_soft_limit + margin_count * _PyOS_STACK_MARGIN_BYTES; + return here_addr <= _tstate->c_stack_soft_limit + margin_count * _PyOS_STACK_MARGIN_BYTES && + here_addr >= _tstate->c_stack_soft_limit - 2 * _PyOS_STACK_MARGIN_BYTES; #else - return here_addr > _tstate->c_stack_soft_limit - margin_count * _PyOS_STACK_MARGIN_BYTES; + return here_addr > _tstate->c_stack_soft_limit - margin_count * _PyOS_STACK_MARGIN_BYTES && + here_addr <= _tstate->c_stack_soft_limit + 2 * _PyOS_STACK_MARGIN_BYTES; #endif } @@ -455,7 +457,7 @@ int pthread_attr_destroy(pthread_attr_t *a) #endif static void -hardware_stack_limits(uintptr_t *base, uintptr_t *top) +hardware_stack_limits(uintptr_t *base, uintptr_t *top, uintptr_t sp) { #ifdef WIN32 ULONG_PTR low, high; @@ -491,10 +493,19 @@ hardware_stack_limits(uintptr_t *base, uintptr_t *top) return; } # endif - uintptr_t here_addr = _Py_get_machine_stack_pointer(); - uintptr_t top_addr = _Py_SIZE_ROUND_UP(here_addr, 4096); + // Add some space for caller function then round to minimum page size + // This is a guess at the top of the stack, but should be a reasonably + // good guess if called from _PyThreadState_Attach when creating a thread. + // If the thread is attached deep in a call stack, then the guess will be poor. +#if _Py_STACK_GROWS_DOWN + uintptr_t top_addr = _Py_SIZE_ROUND_UP(sp + 8*sizeof(void*), SYSTEM_PAGE_SIZE); *top = top_addr; *base = top_addr - Py_C_STACK_SIZE; +# else + uintptr_t base_addr = _Py_SIZE_ROUND_DOWN(sp - 8*sizeof(void*), SYSTEM_PAGE_SIZE); + *base = base_addr; + *top = base_addr + Py_C_STACK_SIZE; +#endif #endif } @@ -543,7 +554,8 @@ void _Py_InitializeRecursionLimits(PyThreadState *tstate) { uintptr_t base, top; - hardware_stack_limits(&base, &top); + uintptr_t here_addr = _Py_get_machine_stack_pointer(); + hardware_stack_limits(&base, &top, here_addr); assert(top != 0); tstate_set_stack(tstate, base, top); @@ -587,7 +599,7 @@ PyUnstable_ThreadState_ResetStackProtection(PyThreadState *tstate) /* The function _Py_EnterRecursiveCallTstate() only calls _Py_CheckRecursiveCall() - if the recursion_depth reaches recursion_limit. */ + if the stack pointer is between the stack base and c_stack_hard_limit. */ int _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where) { @@ -596,10 +608,12 @@ _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where) assert(_tstate->c_stack_soft_limit != 0); assert(_tstate->c_stack_hard_limit != 0); #if _Py_STACK_GROWS_DOWN + assert(here_addr >= _tstate->c_stack_hard_limit - _PyOS_STACK_MARGIN_BYTES); if (here_addr < _tstate->c_stack_hard_limit) { /* Overflowing while handling an overflow. Give up. */ int kbytes_used = (int)(_tstate->c_stack_top - here_addr)/1024; #else + assert(here_addr <= _tstate->c_stack_hard_limit + _PyOS_STACK_MARGIN_BYTES); if (here_addr > _tstate->c_stack_hard_limit) { /* Overflowing while handling an overflow. Give up. */ int kbytes_used = (int)(here_addr - _tstate->c_stack_top)/1024; From 52f70a6f60254fec5297d1ff731b6c1ebc52ec24 Mon Sep 17 00:00:00 2001 From: Guo Ci Date: Wed, 19 Nov 2025 05:30:53 -0500 Subject: [PATCH 103/638] Correct class name from PullDom to PullDOM (#141207) --- Doc/library/xml.dom.pulldom.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/xml.dom.pulldom.rst b/Doc/library/xml.dom.pulldom.rst index 8bceeecd46393e..a21cfaa4645419 100644 --- a/Doc/library/xml.dom.pulldom.rst +++ b/Doc/library/xml.dom.pulldom.rst @@ -74,7 +74,7 @@ given point) or to make use of the :func:`DOMEventStream.expandNode` method and switch to DOM-related processing. -.. class:: PullDom(documentFactory=None) +.. class:: PullDOM(documentFactory=None) Subclass of :class:`xml.sax.handler.ContentHandler`. From afa0badcc587ea7500e2b4dd2ea269f8bbda5fb2 Mon Sep 17 00:00:00 2001 From: da-woods Date: Wed, 19 Nov 2025 11:38:10 +0000 Subject: [PATCH 104/638] gh-141726: Add PyDict_SetDefaultRef() to the Stable ABI (#141727) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/data/stable_abi.dat | 1 + Include/cpython/dictobject.h | 10 ---------- Include/dictobject.h | 12 ++++++++++++ Lib/test/test_stable_abi_ctypes.py | 1 + .../2025-11-18-18-36-15.gh-issue-141726.ILrhyK.rst | 1 + Misc/stable_abi.toml | 2 ++ PC/python3dll.c | 1 + 7 files changed, 18 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-11-18-18-36-15.gh-issue-141726.ILrhyK.rst diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 5cbf3771950fc0..95e032655cf0cc 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -160,6 +160,7 @@ func,PyDict_Merge,3.2,, func,PyDict_MergeFromSeq2,3.2,, func,PyDict_New,3.2,, func,PyDict_Next,3.2,, +func,PyDict_SetDefaultRef,3.15,, func,PyDict_SetItem,3.2,, func,PyDict_SetItemString,3.2,, func,PyDict_Size,3.2,, diff --git a/Include/cpython/dictobject.h b/Include/cpython/dictobject.h index df9ec7050fca1a..5f2f7b6d4f56bd 100644 --- a/Include/cpython/dictobject.h +++ b/Include/cpython/dictobject.h @@ -39,16 +39,6 @@ Py_DEPRECATED(3.14) PyAPI_FUNC(PyObject *) _PyDict_GetItemStringWithError(PyObje PyAPI_FUNC(PyObject *) PyDict_SetDefault( PyObject *mp, PyObject *key, PyObject *defaultobj); -// Inserts `key` with a value `default_value`, if `key` is not already present -// in the dictionary. If `result` is not NULL, then the value associated -// with `key` is returned in `*result` (either the existing value, or the now -// inserted `default_value`). -// Returns: -// -1 on error -// 0 if `key` was not present and `default_value` was inserted -// 1 if `key` was present and `default_value` was not inserted -PyAPI_FUNC(int) PyDict_SetDefaultRef(PyObject *mp, PyObject *key, PyObject *default_value, PyObject **result); - /* Get the number of items of a dictionary. */ static inline Py_ssize_t PyDict_GET_SIZE(PyObject *op) { PyDictObject *mp; diff --git a/Include/dictobject.h b/Include/dictobject.h index 1bbeec1ab699e7..0384e3131dcdb5 100644 --- a/Include/dictobject.h +++ b/Include/dictobject.h @@ -68,6 +68,18 @@ PyAPI_FUNC(int) PyDict_GetItemRef(PyObject *mp, PyObject *key, PyObject **result PyAPI_FUNC(int) PyDict_GetItemStringRef(PyObject *mp, const char *key, PyObject **result); #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030F0000 +// Inserts `key` with a value `default_value`, if `key` is not already present +// in the dictionary. If `result` is not NULL, then the value associated +// with `key` is returned in `*result` (either the existing value, or the now +// inserted `default_value`). +// Returns: +// -1 on error +// 0 if `key` was not present and `default_value` was inserted +// 1 if `key` was present and `default_value` was not inserted +PyAPI_FUNC(int) PyDict_SetDefaultRef(PyObject *mp, PyObject *key, PyObject *default_value, PyObject **result); +#endif + #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030A0000 PyAPI_FUNC(PyObject *) PyObject_GenericGetDict(PyObject *, void *); #endif diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 7167646ecc6734..bc834f5a6816f3 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -165,6 +165,7 @@ def test_windows_feature_macros(self): "PyDict_MergeFromSeq2", "PyDict_New", "PyDict_Next", + "PyDict_SetDefaultRef", "PyDict_SetItem", "PyDict_SetItemString", "PyDict_Size", diff --git a/Misc/NEWS.d/next/C_API/2025-11-18-18-36-15.gh-issue-141726.ILrhyK.rst b/Misc/NEWS.d/next/C_API/2025-11-18-18-36-15.gh-issue-141726.ILrhyK.rst new file mode 100644 index 00000000000000..3fdad5c6b3e8f4 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-11-18-18-36-15.gh-issue-141726.ILrhyK.rst @@ -0,0 +1 @@ +Add :c:func:`PyDict_SetDefaultRef` to the Stable ABI. diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 7ee6cf1dae5a33..5c503f81d3299a 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2639,3 +2639,5 @@ added = '3.15' [const.Py_mod_token] added = '3.15' +[function.PyDict_SetDefaultRef] + added = '3.15' diff --git a/PC/python3dll.c b/PC/python3dll.c index 99e0f05fe03209..35db1a660a762f 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -191,6 +191,7 @@ EXPORT_FUNC(PyDict_Merge) EXPORT_FUNC(PyDict_MergeFromSeq2) EXPORT_FUNC(PyDict_New) EXPORT_FUNC(PyDict_Next) +EXPORT_FUNC(PyDict_SetDefaultRef) EXPORT_FUNC(PyDict_SetItem) EXPORT_FUNC(PyDict_SetItemString) EXPORT_FUNC(PyDict_Size) From 95296a9d40aa2d58502a09e86e2a93c03df23366 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 19 Nov 2025 13:55:10 +0200 Subject: [PATCH 105/638] gh-140875: Fix handling of unclosed charrefs before EOF in HTMLParser (GH-140904) --- Lib/html/parser.py | 29 +++-- Lib/test/test_htmlparser.py | 110 ++++++++++++++---- ...-11-02-10-44-23.gh-issue-140875.wt6B37.rst | 3 + 3 files changed, 109 insertions(+), 33 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-02-10-44-23.gh-issue-140875.wt6B37.rst diff --git a/Lib/html/parser.py b/Lib/html/parser.py index e50620de800d63..80fb8c3f929f6b 100644 --- a/Lib/html/parser.py +++ b/Lib/html/parser.py @@ -24,6 +24,7 @@ entityref = re.compile('&([a-zA-Z][-.a-zA-Z0-9]*)[^a-zA-Z0-9]') charref = re.compile('&#(?:[0-9]+|[xX][0-9a-fA-F]+)[^0-9a-fA-F]') +incomplete_charref = re.compile('&#(?:[0-9]|[xX][0-9a-fA-F])') attr_charref = re.compile(r'&(#[0-9]+|#[xX][0-9a-fA-F]+|[a-zA-Z][a-zA-Z0-9]*)[;=]?') starttagopen = re.compile('<[a-zA-Z]') @@ -304,10 +305,20 @@ def goahead(self, end): k = k - 1 i = self.updatepos(i, k) continue + match = incomplete_charref.match(rawdata, i) + if match: + if end: + self.handle_charref(rawdata[i+2:]) + i = self.updatepos(i, n) + break + # incomplete + break + elif i + 3 < n: # larger than "&#x" + # not the end of the buffer, and can't be confused + # with some other construct + self.handle_data("&#") + i = self.updatepos(i, i + 2) else: - if ";" in rawdata[i:]: # bail by consuming &# - self.handle_data(rawdata[i:i+2]) - i = self.updatepos(i, i+2) break elif startswith('&', i): match = entityref.match(rawdata, i) @@ -321,15 +332,13 @@ def goahead(self, end): continue match = incomplete.match(rawdata, i) if match: - # match.group() will contain at least 2 chars - if end and match.group() == rawdata[i:]: - k = match.end() - if k <= i: - k = n - i = self.updatepos(i, i + 1) + if end: + self.handle_entityref(rawdata[i+1:]) + i = self.updatepos(i, n) + break # incomplete break - elif (i + 1) < n: + elif i + 1 < n: # not the end of the buffer, and can't be confused # with some other construct self.handle_data("&") diff --git a/Lib/test/test_htmlparser.py b/Lib/test/test_htmlparser.py index 19dde9362a43b6..e4eff1ea17a670 100644 --- a/Lib/test/test_htmlparser.py +++ b/Lib/test/test_htmlparser.py @@ -109,12 +109,13 @@ def get_events(self): class TestCaseBase(unittest.TestCase): - def get_collector(self): - return EventCollector(convert_charrefs=False) + def get_collector(self, convert_charrefs=False): + return EventCollector(convert_charrefs=convert_charrefs) - def _run_check(self, source, expected_events, collector=None): + def _run_check(self, source, expected_events, + *, collector=None, convert_charrefs=False): if collector is None: - collector = self.get_collector() + collector = self.get_collector(convert_charrefs=convert_charrefs) parser = collector for s in source: parser.feed(s) @@ -128,7 +129,7 @@ def _run_check(self, source, expected_events, collector=None): def _run_check_extra(self, source, events): self._run_check(source, events, - EventCollectorExtra(convert_charrefs=False)) + collector=EventCollectorExtra(convert_charrefs=False)) class HTMLParserTestCase(TestCaseBase): @@ -187,10 +188,87 @@ def test_malformatted_charref(self): ]) def test_unclosed_entityref(self): - self._run_check("&entityref foo", [ - ("entityref", "entityref"), - ("data", " foo"), - ]) + self._run_check('> <', [('entityref', 'gt'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('> <', [('data', '> <')], convert_charrefs=True) + + self._run_check('&undefined <', + [('entityref', 'undefined'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('&undefined <', [('data', '&undefined <')], + convert_charrefs=True) + + self._run_check('>undefined <', + [('entityref', 'gtundefined'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('>undefined <', [('data', '>undefined <')], + convert_charrefs=True) + + self._run_check('& <', [('data', '& '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('& <', [('data', '& <')], convert_charrefs=True) + + def test_eof_in_entityref(self): + self._run_check('>', [('entityref', 'gt')], convert_charrefs=False) + self._run_check('>', [('data', '>')], convert_charrefs=True) + + self._run_check('&g', [('entityref', 'g')], convert_charrefs=False) + self._run_check('&g', [('data', '&g')], convert_charrefs=True) + + self._run_check('&undefined', [('entityref', 'undefined')], + convert_charrefs=False) + self._run_check('&undefined', [('data', '&undefined')], + convert_charrefs=True) + + self._run_check('>undefined', [('entityref', 'gtundefined')], + convert_charrefs=False) + self._run_check('>undefined', [('data', '>undefined')], + convert_charrefs=True) + + self._run_check('&', [('data', '&')], convert_charrefs=False) + self._run_check('&', [('data', '&')], convert_charrefs=True) + + def test_unclosed_charref(self): + self._run_check('{ <', [('charref', '123'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('{ <', [('data', '{ <')], convert_charrefs=True) + self._run_check('« <', [('charref', 'xab'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('« <', [('data', '\xab <')], convert_charrefs=True) + + self._run_check('� <', + [('charref', '123456789'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('� <', [('data', '\ufffd <')], + convert_charrefs=True) + self._run_check('� <', + [('charref', 'x123456789'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('� <', [('data', '\ufffd <')], + convert_charrefs=True) + + self._run_check('&# <', [('data', '&# '), ('entityref', 'lt')], convert_charrefs=False) + self._run_check('&# <', [('data', '&# <')], convert_charrefs=True) + self._run_check('&#x <', [('data', '&#x '), ('entityref', 'lt')], convert_charrefs=False) + self._run_check('&#x <', [('data', '&#x <')], convert_charrefs=True) + + def test_eof_in_charref(self): + self._run_check('{', [('charref', '123')], convert_charrefs=False) + self._run_check('{', [('data', '{')], convert_charrefs=True) + self._run_check('«', [('charref', 'xab')], convert_charrefs=False) + self._run_check('«', [('data', '\xab')], convert_charrefs=True) + + self._run_check('�', [('charref', '123456789')], + convert_charrefs=False) + self._run_check('�', [('data', '\ufffd')], convert_charrefs=True) + self._run_check('�', [('charref', 'x123456789')], + convert_charrefs=False) + self._run_check('�', [('data', '\ufffd')], convert_charrefs=True) + + self._run_check('&#', [('data', '&#')], convert_charrefs=False) + self._run_check('&#', [('data', '&#')], convert_charrefs=True) + self._run_check('&#x', [('data', '&#x')], convert_charrefs=False) + self._run_check('&#x', [('data', '&#x')], convert_charrefs=True) def test_bad_nesting(self): # Strangely, this *is* supposed to test that overlapping @@ -762,20 +840,6 @@ def test_correct_detection_of_start_tags(self): ] self._run_check(html, expected) - def test_EOF_in_charref(self): - # see #17802 - # This test checks that the UnboundLocalError reported in the issue - # is not raised, however I'm not sure the returned values are correct. - # Maybe HTMLParser should use self.unescape for these - data = [ - ('a&', [('data', 'a&')]), - ('a&b', [('data', 'ab')]), - ('a&b ', [('data', 'a'), ('entityref', 'b'), ('data', ' ')]), - ('a&b;', [('data', 'a'), ('entityref', 'b')]), - ] - for html, expected in data: - self._run_check(html, expected) - def test_eof_in_comments(self): data = [ (' + +
+
🥇
diff --git a/Lib/profiling/sampling/sample.py b/Lib/profiling/sampling/sample.py index f3fa441a35f420..bcc24319aab033 100644 --- a/Lib/profiling/sampling/sample.py +++ b/Lib/profiling/sampling/sample.py @@ -31,6 +31,7 @@ def __init__(self, pid, sample_interval_usec, all_threads, *, mode=PROFILING_MOD self.pid = pid self.sample_interval_usec = sample_interval_usec self.all_threads = all_threads + self.mode = mode # Store mode for later use if _FREE_THREADED_BUILD: self.unwinder = _remote_debugging.RemoteUnwinder( self.pid, all_threads=self.all_threads, mode=mode, native=native, gc=gc, @@ -117,7 +118,7 @@ def sample(self, collector, duration_sec=10): # Pass stats to flamegraph collector if it's the right type if hasattr(collector, 'set_stats'): - collector.set_stats(self.sample_interval_usec, running_time, sample_rate, error_rate) + collector.set_stats(self.sample_interval_usec, running_time, sample_rate, error_rate, mode=self.mode) expected_samples = int(duration_sec / sample_interval_sec) if num_samples < expected_samples and not is_live_mode: diff --git a/Lib/profiling/sampling/stack_collector.py b/Lib/profiling/sampling/stack_collector.py index 51d13a648bfa49..9028a8bebb19b4 100644 --- a/Lib/profiling/sampling/stack_collector.py +++ b/Lib/profiling/sampling/stack_collector.py @@ -62,17 +62,65 @@ def __init__(self, *args, **kwargs): self.stats = {} self._root = {"samples": 0, "children": {}, "threads": set()} self._total_samples = 0 + self._sample_count = 0 # Track actual number of samples (not thread traces) self._func_intern = {} self._string_table = StringTable() self._all_threads = set() - def set_stats(self, sample_interval_usec, duration_sec, sample_rate, error_rate=None): + # Thread status statistics (similar to LiveStatsCollector) + self.thread_status_counts = { + "has_gil": 0, + "on_cpu": 0, + "gil_requested": 0, + "unknown": 0, + "total": 0, + } + self.samples_with_gc_frames = 0 + + # Per-thread statistics + self.per_thread_stats = {} # {thread_id: {has_gil, on_cpu, gil_requested, unknown, total, gc_samples}} + + def collect(self, stack_frames, skip_idle=False): + """Override to track thread status statistics before processing frames.""" + # Increment sample count once per sample + self._sample_count += 1 + + # Collect both aggregate and per-thread statistics using base method + status_counts, has_gc_frame, per_thread_stats = self._collect_thread_status_stats(stack_frames) + + # Merge aggregate status counts + for key in status_counts: + self.thread_status_counts[key] += status_counts[key] + + # Update aggregate GC frame count + if has_gc_frame: + self.samples_with_gc_frames += 1 + + # Merge per-thread statistics + for thread_id, stats in per_thread_stats.items(): + if thread_id not in self.per_thread_stats: + self.per_thread_stats[thread_id] = { + "has_gil": 0, + "on_cpu": 0, + "gil_requested": 0, + "unknown": 0, + "total": 0, + "gc_samples": 0, + } + for key, value in stats.items(): + self.per_thread_stats[thread_id][key] += value + + # Call parent collect to process frames + super().collect(stack_frames, skip_idle=skip_idle) + + def set_stats(self, sample_interval_usec, duration_sec, sample_rate, error_rate=None, mode=None): """Set profiling statistics to include in flamegraph data.""" self.stats = { "sample_interval_usec": sample_interval_usec, "duration_sec": duration_sec, "sample_rate": sample_rate, - "error_rate": error_rate + "error_rate": error_rate, + "mode": mode } def export(self, filename): @@ -117,7 +165,6 @@ def _format_function_name(func): return f"{funcname} ({filename}:{lineno})" def _convert_to_flamegraph_format(self): - """Convert aggregated trie to d3-flamegraph format with string table optimization.""" if self._total_samples == 0: return { "name": self._string_table.intern("No Data"), @@ -178,6 +225,29 @@ def convert_children(children, min_samples): "strings": self._string_table.get_strings() } + # Calculate thread status percentages for display + total_threads = max(1, self.thread_status_counts["total"]) + thread_stats = { + "has_gil_pct": (self.thread_status_counts["has_gil"] / total_threads) * 100, + "on_cpu_pct": (self.thread_status_counts["on_cpu"] / total_threads) * 100, + "gil_requested_pct": (self.thread_status_counts["gil_requested"] / total_threads) * 100, + "gc_pct": (self.samples_with_gc_frames / max(1, self._sample_count)) * 100, + **self.thread_status_counts + } + + # Calculate per-thread statistics with percentages + per_thread_stats_with_pct = {} + total_samples_denominator = max(1, self._sample_count) + for thread_id, stats in self.per_thread_stats.items(): + total = max(1, stats["total"]) + per_thread_stats_with_pct[thread_id] = { + "has_gil_pct": (stats["has_gil"] / total) * 100, + "on_cpu_pct": (stats["on_cpu"] / total) * 100, + "gil_requested_pct": (stats["gil_requested"] / total) * 100, + "gc_pct": (stats["gc_samples"] / total_samples_denominator) * 100, + **stats + } + # If we only have one root child, make it the root to avoid redundant level if len(root_children) == 1: main_child = root_children[0] @@ -185,7 +255,11 @@ def convert_children(children, min_samples): old_name = self._string_table.get_string(main_child["name"]) new_name = f"Program Root: {old_name}" main_child["name"] = self._string_table.intern(new_name) - main_child["stats"] = self.stats + main_child["stats"] = { + **self.stats, + "thread_stats": thread_stats, + "per_thread_stats": per_thread_stats_with_pct + } main_child["threads"] = sorted(list(self._all_threads)) main_child["strings"] = self._string_table.get_strings() return main_child @@ -194,7 +268,11 @@ def convert_children(children, min_samples): "name": self._string_table.intern("Program Root"), "value": total_samples, "children": root_children, - "stats": self.stats, + "stats": { + **self.stats, + "thread_stats": thread_stats, + "per_thread_stats": per_thread_stats_with_pct + }, "threads": sorted(list(self._all_threads)), "strings": self._string_table.get_strings() } diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py index a592f16b367cbc..38665f5a591eec 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py @@ -14,6 +14,15 @@ FlamegraphCollector, ) from profiling.sampling.gecko_collector import GeckoCollector + from profiling.sampling.constants import ( + PROFILING_MODE_WALL, + PROFILING_MODE_CPU, + ) + from _remote_debugging import ( + THREAD_STATUS_HAS_GIL, + THREAD_STATUS_ON_CPU, + THREAD_STATUS_GIL_REQUESTED, + ) except ImportError: raise unittest.SkipTest( "Test only runs when _remote_debugging is available" @@ -657,17 +666,6 @@ def test_gecko_collector_export(self): def test_gecko_collector_markers(self): """Test Gecko profile markers for GIL and CPU state tracking.""" - try: - from _remote_debugging import ( - THREAD_STATUS_HAS_GIL, - THREAD_STATUS_ON_CPU, - THREAD_STATUS_GIL_REQUESTED, - ) - except ImportError: - THREAD_STATUS_HAS_GIL = 1 << 0 - THREAD_STATUS_ON_CPU = 1 << 1 - THREAD_STATUS_GIL_REQUESTED = 1 << 3 - collector = GeckoCollector(1000) # Status combinations for different thread states @@ -894,3 +892,312 @@ def test_pstats_collector_export(self): self.assertEqual(func1_stats[1], 2) # nc (non-recursive calls) self.assertEqual(func1_stats[2], 2.0) # tt (total time) self.assertEqual(func1_stats[3], 2.0) # ct (cumulative time) + + def test_flamegraph_collector_stats_accumulation(self): + """Test that FlamegraphCollector accumulates stats across samples.""" + collector = FlamegraphCollector(sample_interval_usec=1000) + + # First sample + stack_frames_1 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo(1, [("a.py", 1, "func_a")], status=THREAD_STATUS_HAS_GIL), + MockThreadInfo(2, [("b.py", 2, "func_b")], status=THREAD_STATUS_ON_CPU), + ], + ) + ] + collector.collect(stack_frames_1) + self.assertEqual(collector.thread_status_counts["has_gil"], 1) + self.assertEqual(collector.thread_status_counts["on_cpu"], 1) + self.assertEqual(collector.thread_status_counts["total"], 2) + + # Second sample + stack_frames_2 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo(1, [("a.py", 1, "func_a")], status=THREAD_STATUS_GIL_REQUESTED), + MockThreadInfo(2, [("b.py", 2, "func_b")], status=THREAD_STATUS_HAS_GIL), + MockThreadInfo(3, [("c.py", 3, "func_c")], status=THREAD_STATUS_ON_CPU), + ], + ) + ] + collector.collect(stack_frames_2) + + # Should accumulate + self.assertEqual(collector.thread_status_counts["has_gil"], 2) # 1 + 1 + self.assertEqual(collector.thread_status_counts["on_cpu"], 2) # 1 + 1 + self.assertEqual(collector.thread_status_counts["gil_requested"], 1) # 0 + 1 + self.assertEqual(collector.thread_status_counts["total"], 5) # 2 + 3 + + # Test GC sample tracking + stack_frames_gc = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo(1, [("~", 0, "")], status=THREAD_STATUS_HAS_GIL), + ], + ) + ] + collector.collect(stack_frames_gc) + self.assertEqual(collector.samples_with_gc_frames, 1) + + # Another sample without GC + collector.collect(stack_frames_1) + self.assertEqual(collector.samples_with_gc_frames, 1) # Still 1 + + # Another GC sample + collector.collect(stack_frames_gc) + self.assertEqual(collector.samples_with_gc_frames, 2) + + def test_flamegraph_collector_per_thread_stats(self): + """Test per-thread statistics tracking in FlamegraphCollector.""" + collector = FlamegraphCollector(sample_interval_usec=1000) + + # Multiple threads with different states + stack_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo(1, [("a.py", 1, "func_a")], status=THREAD_STATUS_HAS_GIL), + MockThreadInfo(2, [("b.py", 2, "func_b")], status=THREAD_STATUS_ON_CPU), + MockThreadInfo(3, [("c.py", 3, "func_c")], status=THREAD_STATUS_GIL_REQUESTED), + ], + ) + ] + collector.collect(stack_frames) + + # Check per-thread stats + self.assertIn(1, collector.per_thread_stats) + self.assertIn(2, collector.per_thread_stats) + self.assertIn(3, collector.per_thread_stats) + + # Thread 1: has GIL + self.assertEqual(collector.per_thread_stats[1]["has_gil"], 1) + self.assertEqual(collector.per_thread_stats[1]["on_cpu"], 0) + self.assertEqual(collector.per_thread_stats[1]["total"], 1) + + # Thread 2: on CPU + self.assertEqual(collector.per_thread_stats[2]["has_gil"], 0) + self.assertEqual(collector.per_thread_stats[2]["on_cpu"], 1) + self.assertEqual(collector.per_thread_stats[2]["total"], 1) + + # Thread 3: waiting + self.assertEqual(collector.per_thread_stats[3]["gil_requested"], 1) + self.assertEqual(collector.per_thread_stats[3]["total"], 1) + + # Test accumulation across samples + stack_frames_2 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo(1, [("a.py", 2, "func_b")], status=THREAD_STATUS_ON_CPU), + ], + ) + ] + collector.collect(stack_frames_2) + + self.assertEqual(collector.per_thread_stats[1]["has_gil"], 1) + self.assertEqual(collector.per_thread_stats[1]["on_cpu"], 1) + self.assertEqual(collector.per_thread_stats[1]["total"], 2) + + def test_flamegraph_collector_percentage_calculations(self): + """Test that percentage calculations are correct in exported data.""" + collector = FlamegraphCollector(sample_interval_usec=1000) + + # Create scenario: 60% GIL held, 40% not held + for i in range(6): + stack_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo(1, [("a.py", 1, "func")], status=THREAD_STATUS_HAS_GIL), + ], + ) + ] + collector.collect(stack_frames) + + for i in range(4): + stack_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo(1, [("a.py", 1, "func")], status=THREAD_STATUS_ON_CPU), + ], + ) + ] + collector.collect(stack_frames) + + # Export to get calculated percentages + data = collector._convert_to_flamegraph_format() + thread_stats = data["stats"]["thread_stats"] + + self.assertAlmostEqual(thread_stats["has_gil_pct"], 60.0, places=1) + self.assertAlmostEqual(thread_stats["on_cpu_pct"], 40.0, places=1) + self.assertEqual(thread_stats["total"], 10) + + def test_flamegraph_collector_mode_handling(self): + """Test that profiling mode is correctly passed through to exported data.""" + collector = FlamegraphCollector(sample_interval_usec=1000) + + # Collect some data + stack_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo(1, [("a.py", 1, "func")], status=THREAD_STATUS_HAS_GIL), + ], + ) + ] + collector.collect(stack_frames) + + # Set stats with mode + collector.set_stats( + sample_interval_usec=1000, + duration_sec=1.0, + sample_rate=1000.0, + mode=PROFILING_MODE_CPU + ) + + data = collector._convert_to_flamegraph_format() + self.assertEqual(data["stats"]["mode"], PROFILING_MODE_CPU) + + def test_flamegraph_collector_zero_samples_edge_case(self): + """Test that collector handles zero samples gracefully.""" + collector = FlamegraphCollector(sample_interval_usec=1000) + + # Export without collecting any samples + data = collector._convert_to_flamegraph_format() + + # Should return a valid structure with no data + self.assertIn("name", data) + self.assertEqual(data["value"], 0) + self.assertIn("children", data) + self.assertEqual(len(data["children"]), 0) + + def test_flamegraph_collector_json_structure_includes_stats(self): + """Test that exported JSON includes thread_stats and per_thread_stats.""" + collector = FlamegraphCollector(sample_interval_usec=1000) + + # Collect some data with multiple threads + stack_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo(1, [("a.py", 1, "func_a")], status=THREAD_STATUS_HAS_GIL), + MockThreadInfo(2, [("b.py", 2, "func_b")], status=THREAD_STATUS_ON_CPU), + ], + ) + ] + collector.collect(stack_frames) + + # Set stats + collector.set_stats( + sample_interval_usec=1000, + duration_sec=1.0, + sample_rate=1000.0, + mode=PROFILING_MODE_WALL + ) + + # Export and verify structure + data = collector._convert_to_flamegraph_format() + + # Check that stats object exists and contains expected fields + self.assertIn("stats", data) + stats = data["stats"] + + # Verify thread_stats exists and has expected structure + self.assertIn("thread_stats", stats) + thread_stats = stats["thread_stats"] + self.assertIn("has_gil_pct", thread_stats) + self.assertIn("on_cpu_pct", thread_stats) + self.assertIn("gil_requested_pct", thread_stats) + self.assertIn("gc_pct", thread_stats) + self.assertIn("total", thread_stats) + + # Verify per_thread_stats exists and has data for both threads + self.assertIn("per_thread_stats", stats) + per_thread_stats = stats["per_thread_stats"] + self.assertIn(1, per_thread_stats) + self.assertIn(2, per_thread_stats) + + # Check per-thread structure + for thread_id in [1, 2]: + thread_data = per_thread_stats[thread_id] + self.assertIn("has_gil_pct", thread_data) + self.assertIn("on_cpu_pct", thread_data) + self.assertIn("gil_requested_pct", thread_data) + self.assertIn("gc_pct", thread_data) + self.assertIn("total", thread_data) + + def test_flamegraph_collector_per_thread_gc_percentage(self): + """Test that per-thread GC percentage uses total samples as denominator.""" + collector = FlamegraphCollector(sample_interval_usec=1000) + + # Create 10 samples total: + # - Thread 1 appears in all 10 samples, has GC in 2 of them + # - Thread 2 appears in only 5 samples, has GC in 1 of them + + # First 5 samples: both threads, thread 1 has GC in 2 + for i in range(5): + has_gc = i < 2 # First 2 samples have GC for thread 1 + frames_1 = [("~", 0, "")] if has_gc else [("a.py", 1, "func_a")] + stack_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo(1, frames_1, status=THREAD_STATUS_HAS_GIL), + MockThreadInfo(2, [("b.py", 2, "func_b")], status=THREAD_STATUS_ON_CPU), + ], + ) + ] + collector.collect(stack_frames) + + # Next 5 samples: only thread 1, thread 2 appears in first of these with GC + for i in range(5): + if i == 0: + # Thread 2 appears in this sample with GC + stack_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo(1, [("a.py", 1, "func_a")], status=THREAD_STATUS_HAS_GIL), + MockThreadInfo(2, [("~", 0, "")], status=THREAD_STATUS_ON_CPU), + ], + ) + ] + else: + # Only thread 1 + stack_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo(1, [("a.py", 1, "func_a")], status=THREAD_STATUS_HAS_GIL), + ], + ) + ] + collector.collect(stack_frames) + + # Set stats and export + collector.set_stats( + sample_interval_usec=1000, + duration_sec=1.0, + sample_rate=1000.0, + mode=PROFILING_MODE_WALL + ) + + data = collector._convert_to_flamegraph_format() + per_thread_stats = data["stats"]["per_thread_stats"] + + # Thread 1: appeared in 10 samples, had GC in 2 + # GC percentage should be 2/10 = 20% (using total samples, not thread appearances) + self.assertEqual(collector.per_thread_stats[1]["gc_samples"], 2) + self.assertEqual(collector.per_thread_stats[1]["total"], 10) + self.assertAlmostEqual(per_thread_stats[1]["gc_pct"], 20.0, places=1) + + # Thread 2: appeared in 6 samples, had GC in 1 + # GC percentage should be 1/10 = 10% (using total samples, not thread appearances) + self.assertEqual(collector.per_thread_stats[2]["gc_samples"], 1) + self.assertEqual(collector.per_thread_stats[2]["total"], 6) + self.assertAlmostEqual(per_thread_stats[2]["gc_pct"], 10.0, places=1) diff --git a/Misc/NEWS.d/next/Library/2025-11-24-14-05-52.gh-issue-138122.2bbGA8.rst b/Misc/NEWS.d/next/Library/2025-11-24-14-05-52.gh-issue-138122.2bbGA8.rst new file mode 100644 index 00000000000000..5742beeb85c2a7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-24-14-05-52.gh-issue-138122.2bbGA8.rst @@ -0,0 +1,5 @@ +The ``profiling.sampling`` flamegraph profiler now displays thread status +statistics showing the percentage of time threads spend holding the GIL, +running without the GIL, waiting for the GIL, and performing garbage +collection. These statistics help identify GIL contention and thread behavior +patterns. When filtering by thread, the display shows per-thread metrics. From 056d6c5ed90bfed2861098f1e42640d6ea62cac8 Mon Sep 17 00:00:00 2001 From: yihong Date: Sun, 30 Nov 2025 10:49:13 +0800 Subject: [PATCH 222/638] gh-141999: Handle KeyboardInterrupt when sampling in the new tachyon profiler (#142000) --- Lib/profiling/sampling/sample.py | 88 ++++++++++--------- .../test_sampling_profiler/test_profiler.py | 40 +++++++++ ...-11-27-11-39-50.gh-issue-141999._FKGlu.rst | 2 + 3 files changed, 89 insertions(+), 41 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-27-11-39-50.gh-issue-141999._FKGlu.rst diff --git a/Lib/profiling/sampling/sample.py b/Lib/profiling/sampling/sample.py index bcc24319aab033..82c0d3959ba22d 100644 --- a/Lib/profiling/sampling/sample.py +++ b/Lib/profiling/sampling/sample.py @@ -57,50 +57,56 @@ def sample(self, collector, duration_sec=10): last_sample_time = start_time realtime_update_interval = 1.0 # Update every second last_realtime_update = start_time + interrupted = False - while running_time < duration_sec: - # Check if live collector wants to stop - if hasattr(collector, 'running') and not collector.running: - break - - current_time = time.perf_counter() - if next_time < current_time: - try: - stack_frames = self.unwinder.get_stack_trace() - collector.collect(stack_frames) - except ProcessLookupError: - duration_sec = current_time - start_time + try: + while running_time < duration_sec: + # Check if live collector wants to stop + if hasattr(collector, 'running') and not collector.running: break - except (RuntimeError, UnicodeDecodeError, MemoryError, OSError): - collector.collect_failed_sample() - errors += 1 - except Exception as e: - if not self._is_process_running(): - break - raise e from None - - # Track actual sampling intervals for real-time stats - if num_samples > 0: - actual_interval = current_time - last_sample_time - self.sample_intervals.append( - 1.0 / actual_interval - ) # Convert to Hz - self.total_samples += 1 - - # Print real-time statistics if enabled - if ( - self.realtime_stats - and (current_time - last_realtime_update) - >= realtime_update_interval - ): - self._print_realtime_stats() - last_realtime_update = current_time - - last_sample_time = current_time - num_samples += 1 - next_time += sample_interval_sec + current_time = time.perf_counter() + if next_time < current_time: + try: + stack_frames = self.unwinder.get_stack_trace() + collector.collect(stack_frames) + except ProcessLookupError: + duration_sec = current_time - start_time + break + except (RuntimeError, UnicodeDecodeError, MemoryError, OSError): + collector.collect_failed_sample() + errors += 1 + except Exception as e: + if not self._is_process_running(): + break + raise e from None + + # Track actual sampling intervals for real-time stats + if num_samples > 0: + actual_interval = current_time - last_sample_time + self.sample_intervals.append( + 1.0 / actual_interval + ) # Convert to Hz + self.total_samples += 1 + + # Print real-time statistics if enabled + if ( + self.realtime_stats + and (current_time - last_realtime_update) + >= realtime_update_interval + ): + self._print_realtime_stats() + last_realtime_update = current_time + + last_sample_time = current_time + num_samples += 1 + next_time += sample_interval_sec + + running_time = time.perf_counter() - start_time + except KeyboardInterrupt: + interrupted = True running_time = time.perf_counter() - start_time + print("Interrupted by user.") # Clear real-time stats line if it was being displayed if self.realtime_stats and len(self.sample_intervals) > 0: @@ -121,7 +127,7 @@ def sample(self, collector, duration_sec=10): collector.set_stats(self.sample_interval_usec, running_time, sample_rate, error_rate, mode=self.mode) expected_samples = int(duration_sec / sample_interval_sec) - if num_samples < expected_samples and not is_live_mode: + if num_samples < expected_samples and not is_live_mode and not interrupted: print( f"Warning: missed {expected_samples - num_samples} samples " f"from the expected total of {expected_samples} " diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py b/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py index 2d129dc8db56d1..822f559561eb0a 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py @@ -224,6 +224,46 @@ def test_sample_profiler_missed_samples_warning(self): self.assertIn("Warning: missed", result) self.assertIn("samples from the expected total", result) + def test_sample_profiler_keyboard_interrupt(self): + mock_unwinder = mock.MagicMock() + mock_unwinder.get_stack_trace.side_effect = [ + [ + ( + 1, + [ + mock.MagicMock( + filename="test.py", lineno=10, funcname="test_func" + ) + ], + ) + ], + KeyboardInterrupt(), + ] + + with mock.patch( + "_remote_debugging.RemoteUnwinder" + ) as mock_unwinder_class: + mock_unwinder_class.return_value = mock_unwinder + profiler = SampleProfiler( + pid=12345, sample_interval_usec=10000, all_threads=False + ) + mock_collector = mock.MagicMock() + times = [0.0, 0.01, 0.02, 0.03, 0.04] + with mock.patch("time.perf_counter", side_effect=times): + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + try: + profiler.sample(mock_collector, duration_sec=1.0) + except KeyboardInterrupt: + self.fail( + "KeyboardInterrupt was not handled by the profiler" + ) + result = output.getvalue() + self.assertIn("Interrupted by user.", result) + self.assertIn("Captured", result) + self.assertIn("samples", result) + self.assertNotIn("Warning: missed", result) + @force_not_colorized_test_class class TestPrintSampledStats(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2025-11-27-11-39-50.gh-issue-141999._FKGlu.rst b/Misc/NEWS.d/next/Library/2025-11-27-11-39-50.gh-issue-141999._FKGlu.rst new file mode 100644 index 00000000000000..3b54a831b54c3c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-27-11-39-50.gh-issue-141999._FKGlu.rst @@ -0,0 +1,2 @@ +Correctly allow :exc:`KeyboardInterrupt` to stop the process when using +:mod:`!profiling.sampling`. From cd4d0ae75c0a132f4fdc68ad0d043898931ae999 Mon Sep 17 00:00:00 2001 From: Thierry Martos <81799048+ThierryMT@users.noreply.github.com> Date: Sun, 30 Nov 2025 04:47:31 -0800 Subject: [PATCH 223/638] Improve clarity in tutorial introduction (#140669) --- Doc/tutorial/introduction.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index fb491149793cf7..deabac5253051c 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -49,7 +49,7 @@ primary prompt, ``>>>``. (It shouldn't take long.) Numbers ------- -The interpreter acts as a simple calculator: you can type an expression at it +The interpreter acts as a simple calculator: you can type an expression into it and it will write the value. Expression syntax is straightforward: the operators ``+``, ``-``, ``*`` and ``/`` can be used to perform arithmetic; parentheses (``()``) can be used for grouping. From 229ed3dd1f97b2f87629a240b90eddba5ded67bf Mon Sep 17 00:00:00 2001 From: flovent Date: Mon, 1 Dec 2025 05:10:01 +0800 Subject: [PATCH 224/638] gh-142067: Add missing default value for param in `multiprocessing.Pipe`'s doc (GH-142109) --- Doc/library/multiprocessing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index cbc98b256a93a4..b297001f2b544e 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -890,7 +890,7 @@ For an example of the usage of queues for interprocess communication see :ref:`multiprocessing-examples`. -.. function:: Pipe([duplex]) +.. function:: Pipe(duplex=True) Returns a pair ``(conn1, conn2)`` of :class:`~multiprocessing.connection.Connection` objects representing the From 981ce0cf3af68cd8eee41fed19969cf0f2218572 Mon Sep 17 00:00:00 2001 From: Tadej Magajna Date: Mon, 1 Dec 2025 03:14:20 +0100 Subject: [PATCH 225/638] gh-142066: Fix grammar in multiprocessing Pipes and Queues (GH-142121) docs: Fix grammar in multiprocessing Pipes and Queues (gh-142066) --- Doc/library/multiprocessing.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index b297001f2b544e..92605c57527887 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -832,8 +832,8 @@ raising an exception. One difference from other Python queue implementations, is that :mod:`multiprocessing` queues serializes all objects that are put into them using :mod:`pickle`. -The object return by the get method is a re-created object that does not share memory -with the original object. +The object returned by the get method is a re-created object that does not share +memory with the original object. Note that one can also create a shared queue by using a manager object -- see :ref:`multiprocessing-managers`. From 3e2c55749326809a2fc76b9f2cb87a6f89037ebe Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" <68491+gpshead@users.noreply.github.com> Date: Sun, 30 Nov 2025 18:50:05 -0800 Subject: [PATCH 226/638] gh-141473: Document not calling Popen.wait after Popen.communicate times out. (GH-142101) Document not calling Popen.wait after Popen.communicate times out. Closes #141473 --- Doc/library/subprocess.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index 43da804b62beb1..b8dfcc310771fe 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -846,6 +846,11 @@ Instances of the :class:`Popen` class have the following methods: proc.kill() outs, errs = proc.communicate() + After a call to :meth:`~Popen.communicate` raises :exc:`TimeoutExpired`, do + not call :meth:`~Popen.wait`. Use an additional :meth:`~Popen.communicate` + call to finish handling pipes and populate the :attr:`~Popen.returncode` + attribute. + .. note:: The data read is buffered in memory, so do not use this method if the data From b708485d1ac30f793d74b4fe7121e86dd35cda79 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Mon, 1 Dec 2025 11:16:37 +0000 Subject: [PATCH 227/638] Docs: Upgrade Sphinx to 9.0 (#142114) --- Doc/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/requirements.txt b/Doc/requirements.txt index d5f7b473c3aa84..716772b7f28d99 100644 --- a/Doc/requirements.txt +++ b/Doc/requirements.txt @@ -7,7 +7,7 @@ # won't suddenly cause build failures. Updating the version is fine as long # as no warnings are raised by doing so. # Keep this version in sync with ``Doc/conf.py``. -sphinx~=8.2.0 +sphinx~=9.0.0 blurb From d4fa70706c95a5eec4cca340c6232c92168f6cff Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Mon, 1 Dec 2025 13:36:17 +0000 Subject: [PATCH 228/638] gh-139707: Add mechanism for distributors to supply error messages for missing stdlib modules (GH-140783) --- Doc/using/configure.rst | 24 +++++++++ Doc/whatsnew/3.15.rst | 6 +++ Lib/test/test_traceback.py | 23 ++++++++- Lib/traceback.py | 11 +++- Makefile.pre.in | 6 +++ ...-10-30-10-36-15.gh-issue-139707.QJ1FfJ.rst | 4 ++ Tools/build/check_extension_modules.py | 50 +++++++++++++++++++ configure | 18 +++++++ configure.ac | 9 ++++ 9 files changed, 149 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2025-10-30-10-36-15.gh-issue-139707.QJ1FfJ.rst diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index cdadbe51417499..e140ca5d71f555 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -322,6 +322,30 @@ General Options .. versionadded:: 3.11 +.. option:: --with-missing-stdlib-config=FILE + + Path to a `JSON `_ configuration file + containing custom error messages for missing :term:`standard library` modules. + + This option is intended for Python distributors who wish to provide + distribution-specific guidance when users encounter standard library + modules that are missing or packaged separately. + + The JSON file should map missing module names to custom error message strings. + For example, if your distribution packages :mod:`tkinter` and + :mod:`_tkinter` separately and excludes :mod:`!_gdbm` for legal reasons, + the configuration could contain: + + .. code-block:: json + + { + "_gdbm": "The '_gdbm' module is not available in this distribution" + "tkinter": "Install the python-tk package to use tkinter", + "_tkinter": "Install the python-tk package to use tkinter", + } + + .. versionadded:: next + .. option:: --enable-pystats Turn on internal Python performance statistics gathering. diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 4882ddb4310fc2..27e3f23e47c875 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -1247,6 +1247,12 @@ Build changes set to ``no`` or with :option:`!--without-system-libmpdec`. (Contributed by Sergey B Kirpichev in :gh:`115119`.) +* The new configure option :option:`--with-missing-stdlib-config=FILE` allows + distributors to pass a `JSON `_ + configuration file containing custom error messages for :term:`standard library` + modules that are missing or packaged separately. + (Contributed by Stan Ulbrych and Petr Viktorin in :gh:`139707`.) + Porting to Python 3.15 ====================== diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index bf57867a8715c0..3876f1a74bbc1a 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -5051,7 +5051,7 @@ def test_no_site_package_flavour(self): b"or to enable your virtual environment?"), stderr ) - def test_missing_stdlib_package(self): + def test_missing_stdlib_module(self): code = """ import sys sys.stdlib_module_names |= {'spam'} @@ -5061,6 +5061,27 @@ def test_missing_stdlib_package(self): self.assertIn(b"Standard library module 'spam' was not found", stderr) + code = """ + import sys + import traceback + traceback._MISSING_STDLIB_MODULE_MESSAGES = {'spam': "Install 'spam4life' for 'spam'"} + sys.stdlib_module_names |= {'spam'} + import spam + """ + _, _, stderr = assert_python_failure('-S', '-c', code) + + self.assertIn(b"Install 'spam4life' for 'spam'", stderr) + + @unittest.skipIf(sys.platform == "win32", "Non-Windows test") + def test_windows_only_module_error(self): + try: + import msvcrt # noqa: F401 + except ModuleNotFoundError: + formatted = traceback.format_exc() + self.assertIn("Unsupported platform for Windows-only standard library module 'msvcrt'", formatted) + else: + self.fail("ModuleNotFoundError was not raised") + class TestColorizedTraceback(unittest.TestCase): maxDiff = None diff --git a/Lib/traceback.py b/Lib/traceback.py index 9b4b8c7d566fe8..8a3e0f77e765dc 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -14,6 +14,11 @@ from contextlib import suppress +try: + from _missing_stdlib_info import _MISSING_STDLIB_MODULE_MESSAGES +except ImportError: + _MISSING_STDLIB_MODULE_MESSAGES = {} + __all__ = ['extract_stack', 'extract_tb', 'format_exception', 'format_exception_only', 'format_list', 'format_stack', 'format_tb', 'print_exc', 'format_exc', 'print_exception', @@ -1110,7 +1115,11 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, elif exc_type and issubclass(exc_type, ModuleNotFoundError): module_name = getattr(exc_value, "name", None) if module_name in sys.stdlib_module_names: - self._str = f"Standard library module '{module_name}' was not found" + message = _MISSING_STDLIB_MODULE_MESSAGES.get( + module_name, + f"Standard library module {module_name!r} was not found" + ) + self._str = message elif sys.flags.no_site: self._str += (". Site initialization is disabled, did you forget to " + "add the site-packages directory to sys.path " diff --git a/Makefile.pre.in b/Makefile.pre.in index 7b8e7ec0965180..816080faa1f5c3 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1604,6 +1604,11 @@ sharedmods: $(SHAREDMODS) pybuilddir.txt # dependency on BUILDPYTHON ensures that the target is run last .PHONY: checksharedmods checksharedmods: sharedmods $(PYTHON_FOR_BUILD_DEPS) $(BUILDPYTHON) + @if [ -n "@MISSING_STDLIB_CONFIG@" ]; then \ + $(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/check_extension_modules.py --generate-missing-stdlib-info --with-missing-stdlib-config="@MISSING_STDLIB_CONFIG@"; \ + else \ + $(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/check_extension_modules.py --generate-missing-stdlib-info; \ + fi @$(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/check_extension_modules.py .PHONY: rundsymutil @@ -2820,6 +2825,7 @@ libinstall: all $(srcdir)/Modules/xxmodule.c $(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).py $(DESTDIR)$(LIBDEST); \ $(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfig_vars_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).json $(DESTDIR)$(LIBDEST); \ $(INSTALL_DATA) `cat pybuilddir.txt`/build-details.json $(DESTDIR)$(LIBDEST); \ + $(INSTALL_DATA) `cat pybuilddir.txt`/_missing_stdlib_info.py $(DESTDIR)$(LIBDEST); \ $(INSTALL_DATA) $(srcdir)/LICENSE $(DESTDIR)$(LIBDEST)/LICENSE.txt @ # If app store compliance has been configured, apply the patch to the @ # installed library code. The patch has been previously validated against diff --git a/Misc/NEWS.d/next/Build/2025-10-30-10-36-15.gh-issue-139707.QJ1FfJ.rst b/Misc/NEWS.d/next/Build/2025-10-30-10-36-15.gh-issue-139707.QJ1FfJ.rst new file mode 100644 index 00000000000000..d9870d267042af --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-10-30-10-36-15.gh-issue-139707.QJ1FfJ.rst @@ -0,0 +1,4 @@ +Add configure option :option:`--with-missing-stdlib-config=FILE` allows +which distributors to pass a `JSON `_ +configuration file containing custom error messages for missing +:term:`standard library` modules. diff --git a/Tools/build/check_extension_modules.py b/Tools/build/check_extension_modules.py index 668db8df0bd181..f23c1d5286f92a 100644 --- a/Tools/build/check_extension_modules.py +++ b/Tools/build/check_extension_modules.py @@ -23,9 +23,11 @@ import _imp import argparse import enum +import json import logging import os import pathlib +import pprint import re import sys import sysconfig @@ -116,6 +118,18 @@ help="Print a list of module names to stdout and exit", ) +parser.add_argument( + "--generate-missing-stdlib-info", + action="store_true", + help="Generate file with stdlib module info", +) + +parser.add_argument( + "--with-missing-stdlib-config", + metavar="CONFIG_FILE", + help="Path to JSON config file with custom missing module messages", +) + @enum.unique class ModuleState(enum.Enum): @@ -281,6 +295,39 @@ def list_module_names(self, *, all: bool = False) -> set[str]: names.update(WINDOWS_MODULES) return names + def generate_missing_stdlib_info(self, config_path: str | None = None) -> None: + config_messages = {} + if config_path: + try: + with open(config_path, encoding='utf-8') as f: + config_messages = json.load(f) + except (FileNotFoundError, json.JSONDecodeError) as e: + raise RuntimeError(f"Failed to load missing stdlib config {config_path!r}") from e + + messages = {} + for name in WINDOWS_MODULES: + messages[name] = f"Unsupported platform for Windows-only standard library module {name!r}" + + for modinfo in self.modules: + if modinfo.state in (ModuleState.DISABLED, ModuleState.DISABLED_SETUP): + messages[modinfo.name] = f"Standard library module disabled during build {modinfo.name!r} was not found" + elif modinfo.state == ModuleState.NA: + messages[modinfo.name] = f"Unsupported platform for standard library module {modinfo.name!r}" + + messages.update(config_messages) + + content = f'''\ +# Standard library information used by the traceback module for more informative +# ModuleNotFound error messages. +# Generated by check_extension_modules.py + +_MISSING_STDLIB_MODULE_MESSAGES = {pprint.pformat(messages)} +''' + + output_path = self.builddir / "_missing_stdlib_info.py" + with open(output_path, "w", encoding="utf-8") as f: + f.write(content) + def get_builddir(self) -> pathlib.Path: try: with open(self.pybuilddir_txt, encoding="utf-8") as f: @@ -499,6 +546,9 @@ def main() -> None: names = checker.list_module_names(all=True) for name in sorted(names): print(name) + elif args.generate_missing_stdlib_info: + checker.check() + checker.generate_missing_stdlib_info(args.with_missing_stdlib_config) else: checker.check() checker.summary(verbose=args.verbose) diff --git a/configure b/configure index 4bcb639d781dd7..620878bb181378 100755 --- a/configure +++ b/configure @@ -1012,6 +1012,7 @@ UNIVERSALSDK host_exec_prefix host_prefix MACHDEP +MISSING_STDLIB_CONFIG PKG_CONFIG_LIBDIR PKG_CONFIG_PATH PKG_CONFIG @@ -1083,6 +1084,7 @@ ac_user_opts=' enable_option_checking with_build_python with_pkg_config +with_missing_stdlib_config enable_universalsdk with_universal_archs with_framework_name @@ -1862,6 +1864,9 @@ Optional Packages: --with-pkg-config=[yes|no|check] use pkg-config to detect build options (default is check) + --with-missing-stdlib-config=FILE + File with custom module error messages for missing + stdlib modules --with-universal-archs=ARCH specify the kind of macOS universal binary that should be created. This option is only valid when @@ -4095,6 +4100,19 @@ if test "$with_pkg_config" = yes -a -z "$PKG_CONFIG"; then as_fn_error $? "pkg-config is required" "$LINENO" 5] fi + +# Check whether --with-missing-stdlib-config was given. +if test ${with_missing_stdlib_config+y} +then : + withval=$with_missing_stdlib_config; MISSING_STDLIB_CONFIG="$withval" +else case e in #( + e) MISSING_STDLIB_CONFIG="" + ;; +esac +fi + + + # Set name for machine-dependent library files { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking MACHDEP" >&5 diff --git a/configure.ac b/configure.ac index a1f1cf207c5f34..8ef479fe32036c 100644 --- a/configure.ac +++ b/configure.ac @@ -307,6 +307,15 @@ if test "$with_pkg_config" = yes -a -z "$PKG_CONFIG"; then AC_MSG_ERROR([pkg-config is required])] fi +dnl Allow distributors to provide custom missing stdlib module error messages +AC_ARG_WITH([missing-stdlib-config], + [AS_HELP_STRING([--with-missing-stdlib-config=FILE], + [File with custom module error messages for missing stdlib modules])], + [MISSING_STDLIB_CONFIG="$withval"], + [MISSING_STDLIB_CONFIG=""] +) +AC_SUBST([MISSING_STDLIB_CONFIG]) + # Set name for machine-dependent library files AC_ARG_VAR([MACHDEP], [name for machine-dependent library files]) AC_MSG_CHECKING([MACHDEP]) From 5a4c4a033a4a54481be6870aa1896fad732555b5 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 1 Dec 2025 17:26:07 +0200 Subject: [PATCH 229/638] gh-119451: Fix a potential denial of service in http.client (GH-119454) Reading the whole body of the HTTP response could cause OOM if the Content-Length value is too large even if the server does not send a large amount of data. Now the HTTP client reads large data by chunks, therefore the amount of consumed memory is proportional to the amount of sent data. --- Lib/http/client.py | 28 ++++++-- Lib/test/test_httplib.py | 66 +++++++++++++++++++ ...-05-23-11-47-48.gh-issue-119451.qkJe9-.rst | 5 ++ 3 files changed, 95 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2024-05-23-11-47-48.gh-issue-119451.qkJe9-.rst diff --git a/Lib/http/client.py b/Lib/http/client.py index 4b9a61cfc1159f..73c3256734a64f 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -111,6 +111,11 @@ _MAXLINE = 65536 _MAXHEADERS = 100 +# Data larger than this will be read in chunks, to prevent extreme +# overallocation. +_MIN_READ_BUF_SIZE = 1 << 20 + + # Header name/value ABNF (http://tools.ietf.org/html/rfc7230#section-3.2) # # VCHAR = %x21-7E @@ -642,10 +647,25 @@ def _safe_read(self, amt): reading. If the bytes are truly not available (due to EOF), then the IncompleteRead exception can be used to detect the problem. """ - data = self.fp.read(amt) - if len(data) < amt: - raise IncompleteRead(data, amt-len(data)) - return data + cursize = min(amt, _MIN_READ_BUF_SIZE) + data = self.fp.read(cursize) + if len(data) >= amt: + return data + if len(data) < cursize: + raise IncompleteRead(data, amt - len(data)) + + data = io.BytesIO(data) + data.seek(0, 2) + while True: + # This is a geometric increase in read size (never more than + # doubling out the current length of data per loop iteration). + delta = min(cursize, amt - cursize) + data.write(self.fp.read(delta)) + if data.tell() >= amt: + return data.getvalue() + cursize += delta + if data.tell() < cursize: + raise IncompleteRead(data.getvalue(), amt - data.tell()) def _safe_readinto(self, b): """Same as _safe_read, but for reading into a buffer.""" diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 47e3914d1dd62e..44044d0385c72e 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -1511,6 +1511,72 @@ def run_server(): thread.join() self.assertEqual(result, b"proxied data\n") + def test_large_content_length(self): + serv = socket.create_server((HOST, 0)) + self.addCleanup(serv.close) + + def run_server(): + [conn, address] = serv.accept() + with conn: + while conn.recv(1024): + conn.sendall( + b"HTTP/1.1 200 Ok\r\n" + b"Content-Length: %d\r\n" + b"\r\n" % size) + conn.sendall(b'A' * (size//3)) + conn.sendall(b'B' * (size - size//3)) + + thread = threading.Thread(target=run_server) + thread.start() + self.addCleanup(thread.join, 1.0) + + conn = client.HTTPConnection(*serv.getsockname()) + try: + for w in range(15, 27): + size = 1 << w + conn.request("GET", "/") + with conn.getresponse() as response: + self.assertEqual(len(response.read()), size) + finally: + conn.close() + thread.join(1.0) + + def test_large_content_length_truncated(self): + serv = socket.create_server((HOST, 0)) + self.addCleanup(serv.close) + + def run_server(): + while True: + [conn, address] = serv.accept() + with conn: + conn.recv(1024) + if not size: + break + conn.sendall( + b"HTTP/1.1 200 Ok\r\n" + b"Content-Length: %d\r\n" + b"\r\n" + b"Text" % size) + + thread = threading.Thread(target=run_server) + thread.start() + self.addCleanup(thread.join, 1.0) + + conn = client.HTTPConnection(*serv.getsockname()) + try: + for w in range(18, 65): + size = 1 << w + conn.request("GET", "/") + with conn.getresponse() as response: + self.assertRaises(client.IncompleteRead, response.read) + conn.close() + finally: + conn.close() + size = 0 + conn.request("GET", "/") + conn.close() + thread.join(1.0) + def test_putrequest_override_domain_validation(self): """ It should be possible to override the default validation diff --git a/Misc/NEWS.d/next/Security/2024-05-23-11-47-48.gh-issue-119451.qkJe9-.rst b/Misc/NEWS.d/next/Security/2024-05-23-11-47-48.gh-issue-119451.qkJe9-.rst new file mode 100644 index 00000000000000..6d6f25cd2f8bf7 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-05-23-11-47-48.gh-issue-119451.qkJe9-.rst @@ -0,0 +1,5 @@ +Fix a potential memory denial of service in the :mod:`http.client` module. +When connecting to a malicious server, it could cause +an arbitrary amount of memory to be allocated. +This could have led to symptoms including a :exc:`MemoryError`, swapping, out +of memory (OOM) killed processes or containers, or even system crashes. From 694922cf40aa3a28f898b5f5ee08b71b4922df70 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 1 Dec 2025 17:28:15 +0200 Subject: [PATCH 230/638] gh-119342: Fix a potential denial of service in plistlib (GH-119343) Reading a specially prepared small Plist file could cause OOM because file's read(n) preallocates a bytes object for reading the specified amount of data. Now plistlib reads large data by chunks, therefore the upper limit of consumed memory is proportional to the size of the input file. --- Lib/plistlib.py | 31 ++++++++++------ Lib/test/test_plistlib.py | 37 +++++++++++++++++-- ...-05-21-22-11-31.gh-issue-119342.BTFj4Z.rst | 5 +++ 3 files changed, 59 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2024-05-21-22-11-31.gh-issue-119342.BTFj4Z.rst diff --git a/Lib/plistlib.py b/Lib/plistlib.py index 67e832db217319..655c51eea3da5d 100644 --- a/Lib/plistlib.py +++ b/Lib/plistlib.py @@ -73,6 +73,9 @@ PlistFormat = enum.Enum('PlistFormat', 'FMT_XML FMT_BINARY', module=__name__) globals().update(PlistFormat.__members__) +# Data larger than this will be read in chunks, to prevent extreme +# overallocation. +_MIN_READ_BUF_SIZE = 1 << 20 class UID: def __init__(self, data): @@ -508,12 +511,24 @@ def _get_size(self, tokenL): return tokenL + def _read(self, size): + cursize = min(size, _MIN_READ_BUF_SIZE) + data = self._fp.read(cursize) + while True: + if len(data) != cursize: + raise InvalidFileException + if cursize == size: + return data + delta = min(cursize, size - cursize) + data += self._fp.read(delta) + cursize += delta + def _read_ints(self, n, size): - data = self._fp.read(size * n) + data = self._read(size * n) if size in _BINARY_FORMAT: return struct.unpack(f'>{n}{_BINARY_FORMAT[size]}', data) else: - if not size or len(data) != size * n: + if not size: raise InvalidFileException() return tuple(int.from_bytes(data[i: i + size], 'big') for i in range(0, size * n, size)) @@ -573,22 +588,16 @@ def _read_object(self, ref): elif tokenH == 0x40: # data s = self._get_size(tokenL) - result = self._fp.read(s) - if len(result) != s: - raise InvalidFileException() + result = self._read(s) elif tokenH == 0x50: # ascii string s = self._get_size(tokenL) - data = self._fp.read(s) - if len(data) != s: - raise InvalidFileException() + data = self._read(s) result = data.decode('ascii') elif tokenH == 0x60: # unicode string s = self._get_size(tokenL) * 2 - data = self._fp.read(s) - if len(data) != s: - raise InvalidFileException() + data = self._read(s) result = data.decode('utf-16be') elif tokenH == 0x80: # UID diff --git a/Lib/test/test_plistlib.py b/Lib/test/test_plistlib.py index a0c76e5dec5ebe..de2a2fd1fc34bf 100644 --- a/Lib/test/test_plistlib.py +++ b/Lib/test/test_plistlib.py @@ -903,8 +903,7 @@ def test_dump_naive_datetime_with_aware_datetime_option(self): class TestBinaryPlistlib(unittest.TestCase): - @staticmethod - def decode(*objects, offset_size=1, ref_size=1): + def build(self, *objects, offset_size=1, ref_size=1): data = [b'bplist00'] offset = 8 offsets = [] @@ -916,7 +915,11 @@ def decode(*objects, offset_size=1, ref_size=1): len(objects), 0, offset) data.extend(offsets) data.append(tail) - return plistlib.loads(b''.join(data), fmt=plistlib.FMT_BINARY) + return b''.join(data) + + def decode(self, *objects, offset_size=1, ref_size=1): + data = self.build(*objects, offset_size=offset_size, ref_size=ref_size) + return plistlib.loads(data, fmt=plistlib.FMT_BINARY) def test_nonstandard_refs_size(self): # Issue #21538: Refs and offsets are 24-bit integers @@ -1024,6 +1027,34 @@ def test_invalid_binary(self): with self.assertRaises(plistlib.InvalidFileException): plistlib.loads(b'bplist00' + data, fmt=plistlib.FMT_BINARY) + def test_truncated_large_data(self): + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + def check(data): + with open(os_helper.TESTFN, 'wb') as f: + f.write(data) + # buffered file + with open(os_helper.TESTFN, 'rb') as f: + with self.assertRaises(plistlib.InvalidFileException): + plistlib.load(f, fmt=plistlib.FMT_BINARY) + # unbuffered file + with open(os_helper.TESTFN, 'rb', buffering=0) as f: + with self.assertRaises(plistlib.InvalidFileException): + plistlib.load(f, fmt=plistlib.FMT_BINARY) + for w in range(20, 64): + s = 1 << w + # data + check(self.build(b'\x4f\x13' + s.to_bytes(8, 'big'))) + # ascii string + check(self.build(b'\x5f\x13' + s.to_bytes(8, 'big'))) + # unicode string + check(self.build(b'\x6f\x13' + s.to_bytes(8, 'big'))) + # array + check(self.build(b'\xaf\x13' + s.to_bytes(8, 'big'))) + # dict + check(self.build(b'\xdf\x13' + s.to_bytes(8, 'big'))) + # number of objects + check(b'bplist00' + struct.pack('>6xBBQQQ', 1, 1, s, 0, 8)) + def test_load_aware_datetime(self): data = (b'bplist003B\x04>\xd0d\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00' b'\x01\x01\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00' diff --git a/Misc/NEWS.d/next/Security/2024-05-21-22-11-31.gh-issue-119342.BTFj4Z.rst b/Misc/NEWS.d/next/Security/2024-05-21-22-11-31.gh-issue-119342.BTFj4Z.rst new file mode 100644 index 00000000000000..04fd8faca4cf7e --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-05-21-22-11-31.gh-issue-119342.BTFj4Z.rst @@ -0,0 +1,5 @@ +Fix a potential memory denial of service in the :mod:`plistlib` module. +When reading a Plist file received from untrusted source, it could cause +an arbitrary amount of memory to be allocated. +This could have led to symptoms including a :exc:`MemoryError`, swapping, out +of memory (OOM) killed processes or containers, or even system crashes. From 52f9b5f580b6b85dbf08fa23103d17a60455bc20 Mon Sep 17 00:00:00 2001 From: Yashraj Date: Mon, 1 Dec 2025 21:21:50 +0530 Subject: [PATCH 231/638] gh-141004: Document descriptor and dict proxy type objects (GH-141803) Co-authored-by: Peter Bierma Co-authored-by: Victor Stinner --- Doc/c-api/descriptor.rst | 42 ++++++++++++++++++++++++++++++++++++++++ Doc/c-api/dict.rst | 11 +++++++++++ 2 files changed, 53 insertions(+) diff --git a/Doc/c-api/descriptor.rst b/Doc/c-api/descriptor.rst index 22c3b790cc3ec3..313c534545a861 100644 --- a/Doc/c-api/descriptor.rst +++ b/Doc/c-api/descriptor.rst @@ -21,12 +21,46 @@ found in the dictionary of type objects. .. c:function:: PyObject* PyDescr_NewMember(PyTypeObject *type, struct PyMemberDef *meth) +.. c:var:: PyTypeObject PyMemberDescr_Type + + The type object for member descriptor objects created from + :c:type:`PyMemberDef` structures. These descriptors expose fields of a + C struct as attributes on a type, and correspond + to :class:`types.MemberDescriptorType` objects in Python. + + + +.. c:var:: PyTypeObject PyGetSetDescr_Type + + The type object for get/set descriptor objects created from + :c:type:`PyGetSetDef` structures. These descriptors implement attributes + whose value is computed by C getter and setter functions, and are used + for many built-in type attributes. + + .. c:function:: PyObject* PyDescr_NewMethod(PyTypeObject *type, struct PyMethodDef *meth) +.. c:var:: PyTypeObject PyMethodDescr_Type + + The type object for method descriptor objects created from + :c:type:`PyMethodDef` structures. These descriptors expose C functions as + methods on a type, and correspond to :class:`types.MemberDescriptorType` + objects in Python. + + .. c:function:: PyObject* PyDescr_NewWrapper(PyTypeObject *type, struct wrapperbase *wrapper, void *wrapped) +.. c:var:: PyTypeObject PyWrapperDescr_Type + + The type object for wrapper descriptor objects created by + :c:func:`PyDescr_NewWrapper` and :c:func:`PyWrapper_New`. Wrapper + descriptors are used internally to expose special methods implemented + via wrapper structures, and appear in Python as + :class:`types.WrapperDescriptorType` objects. + + .. c:function:: PyObject* PyDescr_NewClassMethod(PyTypeObject *type, PyMethodDef *method) @@ -55,6 +89,14 @@ Built-in descriptors :class:`classmethod` in the Python layer. +.. c:var:: PyTypeObject PyClassMethodDescr_Type + + The type object for C-level class method descriptor objects. + This is the type of the descriptors created for :func:`classmethod` defined in + C extension types, and is the same object as :class:`classmethod` + in Python. + + .. c:function:: PyObject *PyClassMethod_New(PyObject *callable) Create a new :class:`classmethod` object wrapping *callable*. diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index ede1699cfeb653..9c4428ced41b5a 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -43,6 +43,17 @@ Dictionary Objects prevent modification of the dictionary for non-dynamic class types. +.. c:var:: PyTypeObject PyDictProxy_Type + + The type object for mapping proxy objects created by + :c:func:`PyDictProxy_New` and for the read-only ``__dict__`` attribute + of many built-in types. A :c:type:`PyDictProxy_Type` instance provides a + dynamic, read-only view of an underlying dictionary: changes to the + underlying dictionary are reflected in the proxy, but the proxy itself + does not support mutation operations. This corresponds to + :class:`types.MappingProxyType` in Python. + + .. c:function:: void PyDict_Clear(PyObject *p) Empty an existing dictionary of all key-value pairs. From f87eb4d7cd896c8f2d0df1a679bc32a4ac737fbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20Kiss=20Koll=C3=A1r?= Date: Mon, 1 Dec 2025 17:34:14 +0000 Subject: [PATCH 232/638] gh-138122: New Tachyon UI (#142116) Co-authored-by: Pablo Galindo Salgado --- Lib/profiling/sampling/flamegraph.css | 1446 ++++++++++++----- Lib/profiling/sampling/flamegraph.js | 1085 ++++++++----- .../sampling/flamegraph_template.html | 434 +++-- Lib/profiling/sampling/sample.py | 7 +- Lib/profiling/sampling/stack_collector.py | 4 +- .../test_sampling_profiler/test_collectors.py | 2 +- 6 files changed, 1970 insertions(+), 1008 deletions(-) diff --git a/Lib/profiling/sampling/flamegraph.css b/Lib/profiling/sampling/flamegraph.css index 0a6fde2ad329e6..1703815acd9e1d 100644 --- a/Lib/profiling/sampling/flamegraph.css +++ b/Lib/profiling/sampling/flamegraph.css @@ -1,661 +1,1213 @@ -body { - font-family: - "Source Sans Pro", "Lucida Grande", "Lucida Sans Unicode", "Geneva", - "Verdana", sans-serif; +/* ========================================================================== + Flamegraph Viewer - CSS + Python-branded profiler with dark/light theme support + ========================================================================== */ + +/* -------------------------------------------------------------------------- + CSS Variables & Theme System + -------------------------------------------------------------------------- */ + +:root { + /* Typography */ + --font-sans: "Source Sans Pro", "Lucida Grande", "Lucida Sans Unicode", + "Geneva", "Verdana", sans-serif; + --font-mono: 'SF Mono', 'Monaco', 'Consolas', 'Liberation Mono', monospace; + + /* Python brand colors (theme-independent) */ + --python-blue: #3776ab; + --python-blue-light: #4584bb; + --python-blue-lighter: #5592cc; + --python-gold: #ffd43b; + --python-gold-dark: #ffcd02; + --python-gold-light: #ffdc5c; + + /* Heat palette - defined per theme below */ + + /* Layout */ + --sidebar-width: 280px; + --sidebar-collapsed: 44px; + --topbar-height: 52px; + --statusbar-height: 32px; + + /* Transitions */ + --transition-fast: 0.15s ease; + --transition-normal: 0.25s ease; +} + +/* Light theme (default) - Python yellow-to-blue heat palette */ +:root, [data-theme="light"] { + --bg-primary: #ffffff; + --bg-secondary: #f8f9fa; + --bg-tertiary: #e9ecef; + --border: #e9ecef; + --border-subtle: #f0f2f5; + + --text-primary: #2e3338; + --text-secondary: #5a6c7d; + --text-muted: #8b949e; + + --accent: #3776ab; + --accent-hover: #2d5aa0; + --accent-glow: rgba(55, 118, 171, 0.15); + + --shadow-sm: 0 1px 3px rgba(0, 0, 0, 0.08); + --shadow-md: 0 4px 12px rgba(0, 0, 0, 0.1); + --shadow-lg: 0 8px 24px rgba(0, 0, 0, 0.15); + + --header-gradient: linear-gradient(135deg, #3776ab 0%, #4584bb 100%); + + /* Light mode heat palette - blue to yellow to orange to red (cold to hot) */ + --heat-1: #d6e9f8; + --heat-2: #a8d0ef; + --heat-3: #7ba3d1; + --heat-4: #ffe6a8; + --heat-5: #ffd43b; + --heat-6: #ffb84d; + --heat-7: #ff9966; + --heat-8: #ff6347; +} + +/* Dark theme - teal-to-orange heat palette */ +[data-theme="dark"] { + --bg-primary: #0d1117; + --bg-secondary: #161b22; + --bg-tertiary: #21262d; + --border: #30363d; + --border-subtle: #21262d; + + --text-primary: #e6edf3; + --text-secondary: #8b949e; + --text-muted: #6e7681; + + --accent: #58a6ff; + --accent-hover: #79b8ff; + --accent-glow: rgba(88, 166, 255, 0.15); + + --shadow-sm: 0 1px 3px rgba(0, 0, 0, 0.3); + --shadow-md: 0 4px 12px rgba(0, 0, 0, 0.4); + --shadow-lg: 0 8px 24px rgba(0, 0, 0, 0.5); + + --header-gradient: linear-gradient(135deg, #21262d 0%, #30363d 100%); + + /* Dark mode heat palette - dark blue to teal to yellow to orange (cold to hot) */ + --heat-1: #1e3a5f; + --heat-2: #2d5580; + --heat-3: #4a7ba7; + --heat-4: #5a9fa8; + --heat-5: #7ec488; + --heat-6: #c4de6a; + --heat-7: #f4d44d; + --heat-8: #ff6b35; +} + +/* -------------------------------------------------------------------------- + Base Styles + -------------------------------------------------------------------------- */ + +*, *::before, *::after { + box-sizing: border-box; +} + +html, body { margin: 0; padding: 0; - background: #ffffff; - color: #2e3338; - line-height: 1.6; + height: 100%; + overflow: hidden; } -.header { - background: linear-gradient(135deg, #3776ab 0%, #4584bb 100%); - color: white; - padding: 32px 0; - box-shadow: 0 2px 10px rgba(55, 118, 171, 0.2); +body { + font-family: var(--font-sans); + font-size: 14px; + line-height: 1.6; + color: var(--text-primary); + background: var(--bg-primary); + transition: background var(--transition-normal), color var(--transition-normal); } -.header-content { - max-width: 1200px; - margin: 0 auto; - padding: 0 24px; +/* -------------------------------------------------------------------------- + Layout Structure + -------------------------------------------------------------------------- */ + +.app-layout { display: flex; flex-direction: column; + height: 100vh; +} + +.main-content { + display: flex; + flex: 1; + min-height: 0; +} + +/* -------------------------------------------------------------------------- + Top Bar + -------------------------------------------------------------------------- */ + +.top-bar { + height: 56px; + background: var(--header-gradient); + display: flex; align-items: center; - justify-content: center; - text-align: center; - gap: 20px; + padding: 0 16px; + gap: 16px; + flex-shrink: 0; + box-shadow: 0 2px 10px rgba(55, 118, 171, 0.25); + border-bottom: 2px solid var(--python-gold); } -.python-logo { - width: auto; - height: 72px; - margin-bottom: 12px; /* tighter spacing to avoid visual gap */ +/* Brand / Logo */ +.brand { + display: flex; + align-items: center; + gap: 12px; + color: white; + text-decoration: none; flex-shrink: 0; +} + +.brand-logo { display: flex; align-items: center; justify-content: center; + width: 28px; + height: 28px; + flex-shrink: 0; +} + +/* Style the inlined SVG/img inside brand-logo */ +.brand-logo svg, +.brand-logo img { + width: 28px; + height: 28px; + display: block; + object-fit: contain; + filter: drop-shadow(0 1px 2px rgba(0, 0, 0, 0.2)); } -.python-logo img { - height: 72px; - width: auto; - display: block; /* avoid baseline alignment issues */ - vertical-align: middle; - /* subtle shadow that does not affect layout */ - filter: drop-shadow(0 2px 6px rgba(0, 0, 0, 0.1)); +.brand-info { + display: flex; + flex-direction: column; + line-height: 1.15; } -.header-text h1 { - margin: 0; - font-size: 2.5em; - font-weight: 600; - color: white; - text-shadow: 0 2px 4px rgba(0, 0, 0, 0.1); +.brand-text { + font-weight: 700; + font-size: 16px; + letter-spacing: -0.3px; + text-shadow: 0 1px 2px rgba(0, 0, 0, 0.15); } -.header-text .subtitle { - margin: 8px 0 0 0; - font-size: 1.1em; - color: rgba(255, 255, 255, 0.9); - font-weight: 300; +.brand-subtitle { + font-weight: 500; + font-size: 10px; + opacity: 0.9; + text-transform: uppercase; + letter-spacing: 0.5px; } -.header-search { - width: 100%; - max-width: 500px; +.brand-divider { + width: 1px; + height: 16px; + background: rgba(255, 255, 255, 0.3); +} + +/* Search */ +.search-wrapper { + flex: 1; + max-width: 360px; + position: relative; } -.header-search #search-input { +.search-input { width: 100%; - padding: 12px 20px; - border: 2px solid rgba(255, 255, 255, 0.2); - border-radius: 25px; - font-size: 16px; - font-family: inherit; - background: rgba(255, 255, 255, 0.95); + padding: 8px 36px 8px 14px; + font-family: var(--font-sans); + font-size: 13px; color: #2e3338; - transition: all 0.3s ease; - box-shadow: 0 4px 12px rgba(0, 0, 0, 0.1); - backdrop-filter: blur(10px); + background: rgba(255, 255, 255, 0.95); + border: 2px solid rgba(255, 255, 255, 0.3); + border-radius: 20px; + outline: none; + transition: all var(--transition-fast); + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1); } -.header-search #search-input:focus { - outline: none; +.search-input::placeholder { + color: #6c757d; +} + +.search-input:focus { border-color: rgba(255, 255, 255, 0.8); - background: rgba(255, 255, 255, 1); - box-shadow: 0 6px 20px rgba(0, 0, 0, 0.15); - transform: translateY(-2px); + background: white; + box-shadow: 0 4px 16px rgba(0, 0, 0, 0.15); +} + +/* Dark theme search input */ +[data-theme="dark"] .search-input { + color: #e6edf3; + background: rgba(33, 38, 45, 0.95); + border: 2px solid rgba(88, 166, 255, 0.3); + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.3); } -.header-search #search-input::placeholder { +[data-theme="dark"] .search-input::placeholder { + color: #8b949e; +} + +[data-theme="dark"] .search-input:focus { + border-color: rgba(88, 166, 255, 0.6); + background: rgba(33, 38, 45, 1); + box-shadow: 0 4px 16px rgba(88, 166, 255, 0.2); +} + +.search-input.has-matches { + border-color: rgba(40, 167, 69, 0.8); + box-shadow: 0 4px 16px rgba(40, 167, 69, 0.2); +} + +.search-input.no-matches { + border-color: rgba(220, 53, 69, 0.8); + box-shadow: 0 4px 16px rgba(220, 53, 69, 0.2); +} + +.search-clear { + position: absolute; + right: 10px; + top: 50%; + transform: translateY(-50%); + width: 20px; + height: 20px; + padding: 0; + display: none; + align-items: center; + justify-content: center; + font-size: 14px; + line-height: 1; color: #6c757d; + background: transparent; + border: none; + border-radius: 50%; + cursor: pointer; + transition: color var(--transition-fast); } -.stats-section { - background: #ffffff; - padding: 24px 0; - border-bottom: 1px solid #e9ecef; +.search-clear:hover { + color: #2e3338; } -.stats-container { - max-width: 1200px; - margin: 0 auto; - padding: 0 24px; - display: grid; - grid-template-columns: repeat(3, 1fr); - gap: 20px; +[data-theme="dark"] .search-clear { + color: #8b949e; } -/* Compact Thread Stats Bar - Colorful Square Design */ -.thread-stats-bar { - background: rgba(255, 255, 255, 0.95); - padding: 12px 24px; +[data-theme="dark"] .search-clear:hover { + color: #e6edf3; +} + +.search-wrapper.has-value .search-clear { + display: flex; +} + +/* Toolbar */ +.toolbar { + display: flex; + align-items: center; + gap: 6px; + margin-left: auto; +} + +.toolbar-btn { display: flex; align-items: center; justify-content: center; - gap: 16px; - font-size: 13px; - box-shadow: 0 2px 8px rgba(55, 118, 171, 0.2); + width: 32px; + height: 32px; + padding: 0; + font-size: 15px; + color: white; + background: rgba(255, 255, 255, 0.12); + border: 1px solid rgba(255, 255, 255, 0.18); + border-radius: 6px; + cursor: pointer; + transition: all var(--transition-fast); } -.thread-stat-item { - display: inline-flex; +.toolbar-btn:hover { + background: rgba(255, 255, 255, 0.22); + border-color: rgba(255, 255, 255, 0.35); +} + +.toolbar-btn:active { + transform: scale(0.95); +} + +/* -------------------------------------------------------------------------- + Sidebar + -------------------------------------------------------------------------- */ + +.sidebar { + width: var(--sidebar-width); + background: var(--bg-secondary); + border-right: 1px solid var(--border); + display: flex; + flex-direction: column; + flex-shrink: 0; + overflow: hidden; + position: relative; +} + +.sidebar.collapsed { + width: var(--sidebar-collapsed) !important; + transition: width var(--transition-normal); +} + +.sidebar-toggle { + position: absolute; + top: 12px; + right: 10px; + width: 26px; + height: 26px; + display: flex; align-items: center; - gap: 8px; - background: white; - padding: 6px 14px; - border-radius: 4px; - box-shadow: 0 1px 3px rgba(0, 0, 0, 0.08); - transition: all 0.3s ease; - border: 2px solid; - min-width: 115px; justify-content: center; - animation: fadeIn 0.5s ease-out backwards; + color: var(--text-muted); + background: var(--bg-primary); + border: 1px solid var(--border); + border-radius: 6px; + cursor: pointer; + transition: all var(--transition-fast); + z-index: 10; } -.thread-stat-item:nth-child(1) { animation-delay: 0s; } -.thread-stat-item:nth-child(3) { animation-delay: 0.1s; } -.thread-stat-item:nth-child(5) { animation-delay: 0.2s; } -.thread-stat-item:nth-child(7) { animation-delay: 0.3s; } +.sidebar-toggle svg { + transition: transform var(--transition-fast); +} -@keyframes fadeIn { - from { - opacity: 0; - } - to { - opacity: 1; - } +.sidebar-toggle:hover { + color: var(--accent); + border-color: var(--accent); + background: var(--accent-glow); } -@keyframes slideUp { - from { - opacity: 0; - transform: translateY(15px); - } - to { - opacity: 1; - transform: translateY(0); - } +.sidebar.collapsed .sidebar-toggle { + right: 9px; } -@keyframes gentlePulse { - 0%, 100% { box-shadow: 0 2px 8px rgba(55, 118, 171, 0.15); } - 50% { box-shadow: 0 2px 16px rgba(55, 118, 171, 0.4); } +.sidebar.collapsed .sidebar-toggle svg { + transform: rotate(180deg); } -/* Color-coded borders and subtle glow on hover */ -#gil-held-stat { - --stat-color: 40, 167, 69; - border-color: rgb(var(--stat-color)); - background: linear-gradient(135deg, rgba(var(--stat-color), 0.06) 0%, #ffffff 100%); +.sidebar-content { + flex: 1; + overflow-y: auto; + padding: 44px 14px 14px; } -#gil-released-stat { - --stat-color: 220, 53, 69; - border-color: rgb(var(--stat-color)); - background: linear-gradient(135deg, rgba(var(--stat-color), 0.06) 0%, #ffffff 100%); +.sidebar.collapsed .sidebar-content { + display: none; } -#gil-waiting-stat { - --stat-color: 255, 193, 7; - border-color: rgb(var(--stat-color)); - background: linear-gradient(135deg, rgba(var(--stat-color), 0.06) 0%, #ffffff 100%); +.sidebar-resize-handle { + position: absolute; + top: 0; + right: 0; + width: 6px; + height: 100%; + cursor: col-resize; + background: transparent; + transition: background var(--transition-fast); + z-index: 11; } -#gc-stat { - --stat-color: 111, 66, 193; - border-color: rgb(var(--stat-color)); - background: linear-gradient(135deg, rgba(var(--stat-color), 0.06) 0%, #ffffff 100%); +.sidebar-resize-handle:hover, +.sidebar-resize-handle.resizing { + background: var(--python-gold); } -#gil-held-stat:hover, -#gil-released-stat:hover, -#gil-waiting-stat:hover, -#gc-stat:hover { - box-shadow: 0 0 12px rgba(var(--stat-color), 0.4), 0 1px 3px rgba(0, 0, 0, 0.08); +.sidebar-resize-handle::before { + content: ''; + position: absolute; + top: 50%; + left: 50%; + transform: translate(-50%, -50%); + width: 2px; + height: 40px; + background: var(--border); + border-radius: 1px; + opacity: 0; + transition: opacity var(--transition-fast); } -.thread-stat-item .stat-label { - color: #5a6c7d; - font-weight: 600; - font-size: 11px; - letter-spacing: 0.3px; +.sidebar-resize-handle:hover::before { + opacity: 1; } -.thread-stat-item .stat-value { - color: #2e3338; - font-weight: 800; - font-size: 14px; - font-family: 'SF Mono', 'Monaco', 'Consolas', monospace; +.sidebar.collapsed .sidebar-resize-handle { + display: none; } -.thread-stat-separator { - color: rgba(0, 0, 0, 0.15); - font-weight: 300; - font-size: 16px; - position: relative; - z-index: 1; +body.resizing-sidebar { + cursor: col-resize; + user-select: none; } -/* Responsive - stack on small screens */ -@media (max-width: 768px) { - .thread-stats-bar { - flex-wrap: wrap; - gap: 8px; - font-size: 11px; - padding: 10px 16px; - } +/* Sidebar Logo */ +.sidebar-logo { + display: flex; + justify-content: center; + margin-bottom: 16px; +} - .thread-stat-item { - padding: 4px 10px; - } +.sidebar-logo-img { + width: 90px; + height: 90px; + display: flex; + align-items: center; + justify-content: center; +} - .thread-stat-item .stat-label { - font-size: 11px; - } +.sidebar-logo-img svg, +.sidebar-logo-img img { + width: 100%; + height: 100%; + object-fit: contain; +} - .thread-stat-item .stat-value { - font-size: 12px; - } +/* Sidebar sections */ +.sidebar-section { + margin-bottom: 20px; +} - .thread-stat-separator { - display: none; - } +.sidebar-section:last-child { + margin-bottom: 0; } -.stat-card { - background: #ffffff; - border: 1px solid #e9ecef; - border-radius: 12px; - padding: 20px; +.section-title { + font-size: 10px; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.8px; + color: var(--accent); + margin: 0; + flex: 1; +} + +/* Collapsible sections */ +.collapsible .section-header { display: flex; - align-items: flex-start; - gap: 16px; - box-shadow: 0 2px 8px rgba(0, 0, 0, 0.06); - transition: all 0.2s ease; - min-height: 120px; + align-items: center; + width: 100%; + padding: 0 0 8px 0; + margin-bottom: 10px; + background: none; + border: none; + border-bottom: 2px solid var(--python-gold); + cursor: pointer; + transition: all var(--transition-fast); +} + +.collapsible .section-header:hover { + opacity: 0.8; +} + +.section-chevron { + color: var(--text-muted); + transition: transform var(--transition-fast); +} + +.collapsible.collapsed .section-chevron { + transform: rotate(-90deg); +} + +.section-content { + overflow: hidden; + transition: max-height var(--transition-normal), opacity var(--transition-normal); + max-height: 1000px; + opacity: 1; +} + +.collapsible.collapsed .section-content { + max-height: 0; + opacity: 0; + margin-bottom: -10px; +} + +/* -------------------------------------------------------------------------- + Profile Summary Cards + -------------------------------------------------------------------------- */ + +.summary-grid { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 6px; +} + +.summary-card { + display: flex; + align-items: center; + gap: 8px; + padding: 8px 10px; + background: var(--bg-primary); + border: 1px solid var(--border); + border-radius: 8px; + transition: all var(--transition-fast); animation: slideUp 0.4s ease-out backwards; + animation-delay: calc(var(--i, 0) * 0.05s); + overflow: hidden; } -.stat-card:nth-child(1) { animation-delay: 0.1s; } -.stat-card:nth-child(2) { animation-delay: 0.2s; } -.stat-card:nth-child(3) { animation-delay: 0.3s; } +.summary-card:nth-child(1) { --i: 0; } +.summary-card:nth-child(2) { --i: 1; } +.summary-card:nth-child(3) { --i: 2; } +.summary-card:nth-child(4) { --i: 3; } -.stat-card:hover { - box-shadow: 0 4px 16px rgba(0, 0, 0, 0.1); - transform: translateY(-2px); +.summary-card:hover { + border-color: var(--accent); + background: var(--accent-glow); } -.stat-icon { - font-size: 32px; - width: 56px; - height: 56px; +.summary-icon { + font-size: 16px; + width: 28px; + height: 28px; display: flex; align-items: center; justify-content: center; - background: linear-gradient(135deg, #3776ab 0%, #4584bb 100%); - border-radius: 50%; + background: var(--bg-tertiary); + border-radius: 6px; flex-shrink: 0; - box-shadow: 0 2px 8px rgba(55, 118, 171, 0.3); } -.stat-content { +.summary-data { + min-width: 0; flex: 1; + overflow: hidden; } -.stat-label { - font-size: 14px; - color: #5a6c7d; - font-weight: 500; - margin-bottom: 4px; +.summary-value { + font-family: var(--font-mono); + font-size: 13px; + font-weight: 700; + color: var(--accent); + line-height: 1.2; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.summary-label { + font-size: 8px; + font-weight: 600; + color: var(--text-muted); text-transform: uppercase; - letter-spacing: 0.5px; + letter-spacing: 0.2px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; } -.stat-value { - font-size: 16px; +/* Efficiency Bar */ +.efficiency-section { + margin-top: 10px; + padding-top: 10px; + border-top: 1px solid var(--border); +} + +.efficiency-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 5px; +} + +.efficiency-label { + font-size: 9px; + font-weight: 600; + color: var(--text-secondary); + text-transform: uppercase; + letter-spacing: 0.2px; +} + +.efficiency-value { + font-family: var(--font-mono); + font-size: 11px; font-weight: 700; - color: #2e3338; - line-height: 1.3; - margin-bottom: 4px; - word-break: break-word; - overflow-wrap: break-word; + color: var(--accent); } -.stat-file { - font-size: 12px; - color: #8b949e; - font-weight: 400; - margin-bottom: 2px; - font-family: 'SF Mono', 'Monaco', 'Consolas', monospace; - word-break: break-word; - overflow-wrap: break-word; +.efficiency-bar { + height: 6px; + background: var(--bg-tertiary); + border-radius: 3px; + overflow: hidden; } -.stat-detail { - font-size: 12px; - color: #5a6c7d; - font-weight: 400; - line-height: 1.4; - font-family: 'SF Mono', 'Monaco', 'Consolas', monospace; - word-break: break-word; - overflow-wrap: break-word; +.efficiency-fill { + height: 100%; + background: linear-gradient(90deg, #28a745 0%, #20c997 50%, #17a2b8 100%); + border-radius: 3px; + transition: width 0.6s ease-out; + position: relative; + overflow: hidden; } -.controls { - background: #f8f9fa; - border-bottom: 1px solid #e9ecef; - padding: 20px 0; - text-align: center; +.efficiency-fill::after { + content: ''; + position: absolute; + top: 0; + left: -100%; + width: 100%; + height: 100%; + background: linear-gradient( + 90deg, + transparent 0%, + rgba(255, 255, 255, 0.4) 50%, + transparent 100% + ); + animation: shimmer 2s ease-in-out infinite; +} + +@keyframes shimmer { + 0% { left: -100%; } + 100% { left: 100%; } +} + +/* -------------------------------------------------------------------------- + Thread Stats Grid (in Sidebar) + -------------------------------------------------------------------------- */ + +.thread-stats-section { + display: block; } -.controls-content { - max-width: 1200px; - margin: 0 auto; - padding: 0 24px; +.stats-grid { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 8px; +} + +.stat-tile { + background: var(--bg-primary); + border-radius: 8px; + padding: 10px; text-align: center; + border: 2px solid var(--border); + transition: all var(--transition-fast); + animation: fadeIn 0.4s ease-out backwards; + animation-delay: calc(var(--i, 0) * 0.05s); } +.stat-tile:nth-child(1) { --i: 0; } +.stat-tile:nth-child(2) { --i: 1; } +.stat-tile:nth-child(3) { --i: 2; } +.stat-tile:nth-child(4) { --i: 3; } -.controls button { - background: #3776ab; - color: white; - border: none; - padding: 12px 24px; - margin: 0 8px; - border-radius: 6px; - cursor: pointer; +.stat-tile:hover { + transform: translateY(-2px); + box-shadow: var(--shadow-sm); +} + +.stat-tile-value { + font-family: var(--font-mono); + font-size: 16px; + font-weight: 700; + color: var(--text-primary); + line-height: 1.2; +} + +.stat-tile-label { + font-size: 9px; font-weight: 600; - font-size: 14px; - font-family: inherit; - transition: all 0.2s ease; - box-shadow: 0 2px 4px rgba(55, 118, 171, 0.2); + text-transform: uppercase; + letter-spacing: 0.3px; + color: var(--text-muted); + margin-top: 2px; } -.controls button:hover { - background: #2d5aa0; - transform: translateY(-1px); - box-shadow: 0 4px 8px rgba(55, 118, 171, 0.3); +/* Stat tile color variants */ +.stat-tile--green { --tile-color: 40, 167, 69; --tile-text: #28a745; } +.stat-tile--red { --tile-color: 220, 53, 69; --tile-text: #dc3545; } +.stat-tile--yellow { --tile-color: 255, 193, 7; --tile-text: #d39e00; } +.stat-tile--purple { --tile-color: 111, 66, 193; --tile-text: #6f42c1; } + +.stat-tile[class*="--"] { + border-color: rgba(var(--tile-color), 0.4); + background: linear-gradient(135deg, rgba(var(--tile-color), 0.08) 0%, var(--bg-primary) 100%); } +.stat-tile[class*="--"] .stat-tile-value { color: var(--tile-text); } + +/* -------------------------------------------------------------------------- + Hotspot Cards + -------------------------------------------------------------------------- */ -.controls button:active { - transform: translateY(1px); - box-shadow: 0 1px 2px rgba(55, 118, 171, 0.2); +.hotspot { + display: flex; + align-items: flex-start; + gap: 10px; + padding: 10px; + margin-bottom: 8px; + background: var(--bg-primary); + border: 1px solid var(--border); + border-radius: 8px; + cursor: pointer; + transition: all var(--transition-fast); + opacity: 0; + transform: translateY(8px); + box-shadow: var(--shadow-sm); } -.controls button.secondary { - background: #ffd43b; - color: #2e3338; +.hotspot.visible { + opacity: 1; + transform: translateY(0); +} + +.hotspot:hover { + border-color: var(--accent); + box-shadow: var(--shadow-md); + transform: translateY(-2px); } -.controls button.secondary:hover { - background: #ffcd02; +.hotspot.active { + border-color: var(--python-gold); + background: var(--accent-glow); + box-shadow: 0 0 0 3px var(--accent-glow); } -.controls button.secondary:active { - background: #e6b800; +.hotspot:last-child { + margin-bottom: 0; } -.thread-filter-wrapper { - display: none; +.hotspot-rank { + width: 26px; + height: 26px; + border-radius: 50%; + display: flex; align-items: center; - margin-left: 16px; - background: white; - border-radius: 6px; - padding: 4px 8px 4px 12px; - border: 2px solid #3776ab; - transition: all 0.2s ease; + justify-content: center; + font-weight: 700; + font-size: 12px; + flex-shrink: 0; + background: linear-gradient(135deg, var(--python-blue) 0%, var(--python-blue-light) 100%); + color: white; + box-shadow: 0 2px 4px rgba(55, 118, 171, 0.3); } -.thread-filter-wrapper:hover { - border-color: #2d5aa0; - box-shadow: 0 2px 6px rgba(55, 118, 171, 0.2); +.hotspot-rank--1 { background: linear-gradient(135deg, #d4af37, #f4d03f); color: #5a4a00; } +.hotspot-rank--2 { background: linear-gradient(135deg, #a8a8a8, #c0c0c0); color: #4a4a4a; } +.hotspot-rank--3 { background: linear-gradient(135deg, #cd7f32, #e6a55a); color: #5a3d00; } + +.hotspot-info { + flex: 1; + min-width: 0; } -.thread-filter-label { - color: #3776ab; - font-size: 14px; +.hotspot-func { + font-family: var(--font-mono); + font-size: 11px; font-weight: 600; - margin-right: 8px; + color: var(--text-primary); + line-height: 1.3; + word-break: break-word; + margin-bottom: 2px; +} + +.hotspot-file { + font-family: var(--font-mono); + font-size: 10px; + color: var(--text-muted); + margin-bottom: 3px; + word-break: break-all; +} + +.hotspot-stats { + font-family: var(--font-mono); + font-size: 10px; + color: var(--text-secondary); +} + +.hotspot-percent { + color: var(--accent); + font-weight: 600; +} + +/* -------------------------------------------------------------------------- + Legend + -------------------------------------------------------------------------- */ + +.legend-section { + margin-top: auto; + padding-top: 12px; +} + +.legend { + display: flex; + flex-direction: column; + gap: 4px; +} + +.legend-item { display: flex; align-items: center; + gap: 8px; + padding: 5px 8px; + background: var(--bg-primary); + border-radius: 4px; + border: 1px solid var(--border-subtle); + font-size: 11px; } -.thread-filter-select { - background: transparent; - color: #2e3338; - border: none; - padding: 4px 24px 4px 4px; - font-size: 14px; +.legend-color { + width: 20px; + height: 10px; + border-radius: 2px; + flex-shrink: 0; + border: 1px solid rgba(0, 0, 0, 0.08); +} + +.legend-label { + color: var(--text-primary); + font-weight: 500; + flex: 1; +} + +.legend-range { + font-family: var(--font-mono); + font-size: 9px; + color: var(--text-muted); +} + +/* -------------------------------------------------------------------------- + Thread Filter + -------------------------------------------------------------------------- */ + +.filter-section { + padding-top: 12px; + border-top: 1px solid var(--border); +} + +.filter-label { + display: block; + font-size: 10px; font-weight: 600; + color: var(--text-muted); + margin-bottom: 6px; +} + +.filter-select { + width: 100%; + padding: 7px 28px 7px 10px; + font-family: var(--font-mono); + font-size: 11px; + color: var(--text-primary); + background: var(--bg-primary); + border: 2px solid var(--accent); + border-radius: 6px; cursor: pointer; - min-width: 120px; - font-family: inherit; + outline: none; appearance: none; - -webkit-appearance: none; - -moz-appearance: none; background-image: url("data:image/svg+xml;charset=UTF-8,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24' fill='none' stroke='%233776ab' stroke-width='2' stroke-linecap='round' stroke-linejoin='round'%3e%3cpolyline points='6 9 12 15 18 9'%3e%3c/polyline%3e%3c/svg%3e"); background-repeat: no-repeat; - background-position: right 4px center; - background-size: 16px; + background-position: right 6px center; + background-size: 14px; + transition: all var(--transition-fast); } -.thread-filter-select:focus { - outline: none; +.filter-select:hover { + border-color: var(--accent-hover); + box-shadow: 0 2px 6px var(--accent-glow); } -.thread-filter-select:hover { - color: #3776ab; +.filter-select:focus { + border-color: var(--accent); + box-shadow: 0 0 0 3px var(--accent-glow); } -.thread-filter-select option { - padding: 8px; - background: white; - color: #2e3338; - font-weight: normal; +/* -------------------------------------------------------------------------- + Chart Area + -------------------------------------------------------------------------- */ + +.chart-area { + flex: 1; + min-width: 0; + overflow: hidden; + background: var(--bg-primary); + position: relative; } #chart { width: 100%; - height: calc(100vh - 160px); - overflow: hidden; - background: #ffffff; - padding: 0 40px; + height: 100%; + padding: 16px; + overflow: auto; } +/* D3 Flamegraph overrides */ .d3-flame-graph rect { - /* Prefer selector specificity instead of !important */ stroke: rgba(55, 118, 171, 0.3); stroke-width: 1px; cursor: pointer; - transition: all 0.1s ease; + transition: filter 0.1s ease; } .d3-flame-graph rect:hover { - stroke: #3776ab; + stroke: var(--python-blue); stroke-width: 2px; - filter: brightness(1.05); + filter: brightness(1.08); } .d3-flame-graph text { - /* Ensure labels use our font without !important */ - font-family: "Source Sans Pro", sans-serif; + font-family: var(--font-sans); font-size: 12px; font-weight: 500; - fill: #2e3338; + fill: var(--text-primary); pointer-events: none; } -.info-panel { - position: fixed; - bottom: 24px; - left: 84px; /* Leave space for the button */ - background: white; - padding: 24px; - border-radius: 8px; - border: 1px solid #e9ecef; - font-size: 14px; - max-width: 280px; - box-shadow: 0 4px 20px rgba(0, 0, 0, 0.1); - z-index: 1000; - display: none; +/* Search highlight */ +.d3-flame-graph rect.search-match { + stroke: #ff6b35 !important; + stroke-width: 2px !important; + stroke-dasharray: 4 2; } -.info-panel h3 { - margin: 0 0 16px 0; - color: #3776ab; - font-weight: 600; - font-size: 16px; - border-bottom: 2px solid #ffd43b; - padding-bottom: 8px; +.d3-flame-graph rect.search-dim { + opacity: 0.25; } -.info-panel p { - margin: 12px 0; - color: #5a6c7d; - line-height: 1.5; -} +/* -------------------------------------------------------------------------- + Status Bar + -------------------------------------------------------------------------- */ -.info-panel strong { - color: #3776ab; +.status-bar { + height: var(--statusbar-height); + background: var(--bg-secondary); + border-top: 1px solid var(--border); + display: flex; + align-items: center; + padding: 0 16px; + gap: 16px; + font-family: var(--font-mono); + font-size: 11px; + color: var(--text-secondary); + flex-shrink: 0; } -#show-info-btn { - position: fixed; - bottom: 32px; - left: 32px; - z-index: 1100; - width: 44px; - height: 44px; - border-radius: 50%; - background: #3776ab; - color: white; - border: none; - font-size: 24px; - box-shadow: 0 2px 8px rgba(55, 118, 171, 0.15); - cursor: pointer; +.status-item { display: flex; align-items: center; - justify-content: center; - transition: background 0.2s, transform 0.2s; - animation: gentlePulse 3s ease-in-out infinite; + gap: 5px; } -#show-info-btn:hover { - background: #2d5aa0; - animation: none; - transform: scale(1.05); +.status-item::before { + content: ''; + width: 4px; + height: 4px; + background: var(--python-gold); + border-radius: 50%; } -#close-info-btn { - position: absolute; - top: 8px; - right: 12px; - background: none; - border: none; - font-size: 20px; - cursor: pointer; - color: #3776ab; +.status-item:first-child::before { + display: none; } -@media (max-width: 600px) { - .python-logo { height: 48px; } - .python-logo img { height: 48px; } - #show-info-btn { - left: 8px; - bottom: 8px; - } - .info-panel { - left: 60px; /* Still leave space for button */ - bottom: 8px; - max-width: 90vw; - } +.status-label { + color: var(--text-muted); } -.legend-panel { - position: fixed; - top: 24px; - left: 24px; - background: white; - padding: 24px; +.status-value { + color: var(--text-primary); + font-weight: 500; +} + +.status-value.accent { + color: var(--accent); + font-weight: 600; +} + +/* -------------------------------------------------------------------------- + Tooltip + -------------------------------------------------------------------------- */ + +.python-tooltip { + position: absolute; + z-index: 1000; + pointer-events: none; + background: var(--bg-primary); + border: 1px solid var(--border); border-radius: 8px; - border: 1px solid #e9ecef; + padding: 14px; + max-width: 480px; + box-shadow: var(--shadow-lg); + font-family: var(--font-sans); + font-size: 13px; + color: var(--text-primary); + word-wrap: break-word; + overflow-wrap: break-word; + line-height: 1.5; +} + +.tooltip-header { + margin-bottom: 10px; +} + +.tooltip-title { font-size: 14px; - max-width: 320px; - box-shadow: 0 4px 20px rgba(0, 0, 0, 0.1); - display: none; - z-index: 1001; + font-weight: 600; + color: var(--accent); + line-height: 1.3; + word-break: break-word; + margin-bottom: 4px; } -.legend-panel h3 { - margin: 0 0 20px 0; - color: #3776ab; +.tooltip-location { + font-family: var(--font-mono); + font-size: 11px; + color: var(--text-muted); + background: var(--bg-tertiary); + padding: 4px 8px; + border-radius: 4px; + word-break: break-all; +} + +.tooltip-stats { + display: grid; + grid-template-columns: auto 1fr; + gap: 4px 14px; + font-size: 12px; +} + +.tooltip-stat-label { + color: var(--text-secondary); + font-weight: 500; +} + +.tooltip-stat-value { + color: var(--text-primary); font-weight: 600; - font-size: 18px; - text-align: center; - border-bottom: 2px solid #ffd43b; - padding-bottom: 8px; } -.legend-item { - display: flex; - align-items: center; - margin: 12px 0; - padding: 10px; - border-radius: 6px; - background: #f8f9fa; - border: 1px solid #e9ecef; +.tooltip-stat-value.accent { + color: var(--accent); } -.legend-color { - width: 28px; - height: 18px; - border-radius: 4px; - margin-right: 16px; - border: 1px solid rgba(0, 0, 0, 0.1); - flex-shrink: 0; +.tooltip-source { + margin-top: 10px; + padding-top: 10px; + border-top: 1px solid var(--border); } -.legend-label { - color: #2e3338; +.tooltip-source-title { + font-size: 11px; font-weight: 600; - flex: 1; + color: var(--accent); + margin-bottom: 6px; } -.legend-description { - color: #5a6c7d; - font-size: 12px; - margin-top: 2px; - font-weight: 400; +.tooltip-source-code { + font-family: var(--font-mono); + font-size: 10px; + line-height: 1.5; + background: var(--bg-tertiary); + border: 1px solid var(--border); + border-radius: 6px; + padding: 8px; + max-height: 140px; + overflow-y: auto; + white-space: pre-wrap; + word-break: break-all; } -.chart-container { - background: #ffffff; - margin: 0; - padding: 12px 0; +.tooltip-source-line { + color: var(--text-secondary); + padding: 1px 0; } -/* Tooltip overflow fixes */ -.python-tooltip { - max-width: 500px !important; - word-wrap: break-word !important; - overflow-wrap: break-word !important; - box-sizing: border-box !important; -} - -/* Responsive tooltip adjustments */ -@media (max-width: 768px) { - .python-tooltip { - max-width: calc(100vw - 40px) !important; - max-height: calc(100vh - 80px) !important; - overflow-y: auto !important; - } +.tooltip-source-line.current { + color: var(--accent); + font-weight: 600; } -@media (max-width: 480px) { - .python-tooltip { - max-width: calc(100vw - 20px) !important; - font-size: 12px !important; +.tooltip-hint { + margin-top: 10px; + padding-top: 8px; + border-top: 1px solid var(--border); + font-size: 11px; + color: var(--text-muted); + text-align: center; +} + +/* -------------------------------------------------------------------------- + Animations + -------------------------------------------------------------------------- */ + +@keyframes fadeIn { + from { opacity: 0; } + to { opacity: 1; } +} + +@keyframes slideUp { + from { + opacity: 0; + transform: translateY(12px); + } + to { + opacity: 1; + transform: translateY(0); } } -/* Accessibility: visible focus states */ +/* -------------------------------------------------------------------------- + Focus States (Accessibility) + -------------------------------------------------------------------------- */ + button:focus-visible, select:focus-visible, input:focus-visible { - outline: 2px solid #ffd43b; + outline: 2px solid var(--python-gold); outline-offset: 2px; } -/* Smooth panel transitions */ -.legend-panel, -.info-panel { - transition: opacity 0.2s ease, transform 0.2s ease; +/* -------------------------------------------------------------------------- + Responsive + -------------------------------------------------------------------------- */ + +@media (max-width: 900px) { + .sidebar { + position: fixed; + left: 0; + top: var(--topbar-height); + bottom: var(--statusbar-height); + z-index: 100; + box-shadow: var(--shadow-lg); + } + + .sidebar.collapsed { + width: var(--sidebar-collapsed); + } + + .brand-subtitle { + display: none; + } + + .search-wrapper { + max-width: 220px; + } } -.legend-panel[style*="block"], -.info-panel[style*="block"] { - animation: slideUp 0.2s ease-out; +@media (max-width: 600px) { + .toolbar-btn:not(.theme-toggle) { + display: none; + } + + .search-wrapper { + max-width: 160px; + } + + .brand-info { + display: none; + } + + .stats-grid { + grid-template-columns: 1fr; + } } diff --git a/Lib/profiling/sampling/flamegraph.js b/Lib/profiling/sampling/flamegraph.js index 7faac0effbc561..7a2b2ef2e3135e 100644 --- a/Lib/profiling/sampling/flamegraph.js +++ b/Lib/profiling/sampling/flamegraph.js @@ -5,93 +5,219 @@ let stringTable = []; let originalData = null; let currentThreadFilter = 'all'; -// Function to resolve string indices to actual strings +// Heat colors are now defined in CSS variables (--heat-1 through --heat-8) +// and automatically switch with theme changes - no JS color arrays needed! + +// ============================================================================ +// String Resolution +// ============================================================================ + function resolveString(index) { - if (typeof index === 'number' && index >= 0 && index < stringTable.length) { - return stringTable[index]; - } - // Fallback for non-indexed strings or invalid indices - return String(index); + if (index === null || index === undefined) { + return null; + } + if (typeof index === 'number' && index >= 0 && index < stringTable.length) { + return stringTable[index]; + } + return String(index); } -// Function to recursively resolve all string indices in flamegraph data function resolveStringIndices(node) { - if (!node) return node; + if (!node) return node; - // Create a copy to avoid mutating the original - const resolved = { ...node }; + const resolved = { ...node }; - // Resolve string fields - if (typeof resolved.name === 'number') { - resolved.name = resolveString(resolved.name); - } - if (typeof resolved.filename === 'number') { - resolved.filename = resolveString(resolved.filename); + if (typeof resolved.name === 'number') { + resolved.name = resolveString(resolved.name); + } + if (typeof resolved.filename === 'number') { + resolved.filename = resolveString(resolved.filename); + } + if (typeof resolved.funcname === 'number') { + resolved.funcname = resolveString(resolved.funcname); + } + + if (Array.isArray(resolved.source)) { + resolved.source = resolved.source.map(index => + typeof index === 'number' ? resolveString(index) : index + ); + } + + if (Array.isArray(resolved.children)) { + resolved.children = resolved.children.map(child => resolveStringIndices(child)); + } + + return resolved; +} + +// ============================================================================ +// Theme & UI Controls +// ============================================================================ + +function toggleTheme() { + const html = document.documentElement; + const current = html.getAttribute('data-theme') || 'light'; + const next = current === 'light' ? 'dark' : 'light'; + html.setAttribute('data-theme', next); + localStorage.setItem('flamegraph-theme', next); + + // Update theme button icon + const btn = document.getElementById('theme-btn'); + if (btn) { + btn.innerHTML = next === 'dark' ? '☼' : '☾'; // sun or moon + } + + // Re-render flamegraph with new theme colors + if (window.flamegraphData && originalData) { + const tooltip = createPythonTooltip(originalData); + const chart = createFlamegraph(tooltip, originalData.value); + renderFlamegraph(chart, window.flamegraphData); + } +} + +function toggleSidebar() { + const sidebar = document.getElementById('sidebar'); + if (sidebar) { + const isCollapsing = !sidebar.classList.contains('collapsed'); + + if (isCollapsing) { + // Save current width before collapsing + const currentWidth = sidebar.offsetWidth; + sidebar.dataset.expandedWidth = currentWidth; + localStorage.setItem('flamegraph-sidebar-width', currentWidth); + } else { + // Restore width when expanding + const savedWidth = sidebar.dataset.expandedWidth || localStorage.getItem('flamegraph-sidebar-width'); + if (savedWidth) { + sidebar.style.width = savedWidth + 'px'; + } } - if (typeof resolved.funcname === 'number') { - resolved.funcname = resolveString(resolved.funcname); + + sidebar.classList.toggle('collapsed'); + localStorage.setItem('flamegraph-sidebar', sidebar.classList.contains('collapsed') ? 'collapsed' : 'expanded'); + + // Resize chart after sidebar animation + setTimeout(() => { + resizeChart(); + }, 300); + } +} + +function resizeChart() { + if (window.flamegraphChart && window.flamegraphData) { + const chartArea = document.querySelector('.chart-area'); + if (chartArea) { + window.flamegraphChart.width(chartArea.clientWidth - 32); + d3.select("#chart").datum(window.flamegraphData).call(window.flamegraphChart); } + } +} - // Resolve source lines if present - if (Array.isArray(resolved.source)) { - resolved.source = resolved.source.map(index => - typeof index === 'number' ? resolveString(index) : index - ); +function toggleSection(sectionId) { + const section = document.getElementById(sectionId); + if (section) { + section.classList.toggle('collapsed'); + // Save state + const collapsedSections = JSON.parse(localStorage.getItem('flamegraph-collapsed-sections') || '{}'); + collapsedSections[sectionId] = section.classList.contains('collapsed'); + localStorage.setItem('flamegraph-collapsed-sections', JSON.stringify(collapsedSections)); + } +} + +function restoreUIState() { + // Restore theme + const savedTheme = localStorage.getItem('flamegraph-theme'); + if (savedTheme) { + document.documentElement.setAttribute('data-theme', savedTheme); + const btn = document.getElementById('theme-btn'); + if (btn) { + btn.innerHTML = savedTheme === 'dark' ? '☼' : '☾'; } + } - // Recursively resolve children - if (Array.isArray(resolved.children)) { - resolved.children = resolved.children.map(child => resolveStringIndices(child)); + // Restore sidebar state + const savedSidebar = localStorage.getItem('flamegraph-sidebar'); + if (savedSidebar === 'collapsed') { + const sidebar = document.getElementById('sidebar'); + if (sidebar) sidebar.classList.add('collapsed'); + } + + // Restore sidebar width + const savedWidth = localStorage.getItem('flamegraph-sidebar-width'); + if (savedWidth) { + const sidebar = document.getElementById('sidebar'); + if (sidebar) { + sidebar.style.width = savedWidth + 'px'; } + } - return resolved; + // Restore collapsed sections + const collapsedSections = JSON.parse(localStorage.getItem('flamegraph-collapsed-sections') || '{}'); + for (const [sectionId, isCollapsed] of Object.entries(collapsedSections)) { + if (isCollapsed) { + const section = document.getElementById(sectionId); + if (section) section.classList.add('collapsed'); + } + } } -// Python color palette - cold to hot -const pythonColors = [ - "#fff4bf", // Coldest - light yellow (<1%) - "#ffec9e", // Cold - yellow (1-3%) - "#ffe47d", // Cool - golden yellow (3-6%) - "#ffdc5c", // Medium - golden (6-12%) - "#ffd43b", // Warm - Python gold (12-18%) - "#5592cc", // Hot - light blue (18-35%) - "#4584bb", // Very hot - medium blue (35-60%) - "#3776ab", // Hottest - Python blue (≥60%) -]; - -function ensureLibraryLoaded() { - if (typeof flamegraph === "undefined") { - console.error("d3-flame-graph library not loaded"); - document.getElementById("chart").innerHTML = - '

Error: d3-flame-graph library failed to load

'; - throw new Error("d3-flame-graph library failed to load"); +// ============================================================================ +// Status Bar +// ============================================================================ + +function updateStatusBar(nodeData, rootValue) { + const funcname = resolveString(nodeData.funcname) || resolveString(nodeData.name) || "--"; + const filename = resolveString(nodeData.filename) || ""; + const lineno = nodeData.lineno; + const timeMs = (nodeData.value / 1000).toFixed(2); + const percent = rootValue > 0 ? ((nodeData.value / rootValue) * 100).toFixed(1) : "0.0"; + + const locationEl = document.getElementById('status-location'); + const funcItem = document.getElementById('status-func-item'); + const timeItem = document.getElementById('status-time-item'); + const percentItem = document.getElementById('status-percent-item'); + + if (locationEl) locationEl.style.display = filename && filename !== "~" ? 'flex' : 'none'; + if (funcItem) funcItem.style.display = 'flex'; + if (timeItem) timeItem.style.display = 'flex'; + if (percentItem) percentItem.style.display = 'flex'; + + const fileEl = document.getElementById('status-file'); + if (fileEl && filename && filename !== "~") { + const basename = filename.split('/').pop(); + fileEl.textContent = lineno ? `${basename}:${lineno}` : basename; } + + const funcEl = document.getElementById('status-func'); + if (funcEl) funcEl.textContent = funcname.length > 40 ? funcname.substring(0, 37) + '...' : funcname; + + const timeEl = document.getElementById('status-time'); + if (timeEl) timeEl.textContent = `${timeMs} ms`; + + const percentEl = document.getElementById('status-percent'); + if (percentEl) percentEl.textContent = `${percent}%`; +} + +function clearStatusBar() { + const ids = ['status-location', 'status-func-item', 'status-time-item', 'status-percent-item']; + ids.forEach(id => { + const el = document.getElementById(id); + if (el) el.style.display = 'none'; + }); } +// ============================================================================ +// Tooltip +// ============================================================================ + function createPythonTooltip(data) { const pythonTooltip = flamegraph.tooltip.defaultFlamegraphTooltip(); + pythonTooltip.show = function (d, element) { if (!this._tooltip) { - this._tooltip = d3 - .select("body") + this._tooltip = d3.select("body") .append("div") .attr("class", "python-tooltip") - .style("position", "absolute") - .style("padding", "20px") - .style("background", "white") - .style("color", "#2e3338") - .style("border-radius", "8px") - .style("font-size", "14px") - .style("border", "1px solid #e9ecef") - .style("box-shadow", "0 8px 30px rgba(0, 0, 0, 0.15)") - .style("z-index", "1000") - .style("pointer-events", "none") - .style("font-weight", "400") - .style("line-height", "1.5") - .style("max-width", "500px") - .style("word-wrap", "break-word") - .style("overflow-wrap", "break-word") - .style("font-family", "'Source Sans Pro', sans-serif") .style("opacity", 0); } @@ -101,163 +227,153 @@ function createPythonTooltip(data) { const childCount = d.children ? d.children.length : 0; const source = d.data.source; - // Create source code section if available + const funcname = resolveString(d.data.funcname) || resolveString(d.data.name); + const filename = resolveString(d.data.filename) || ""; + const isSpecialFrame = filename === "~"; + + // Build source section let sourceSection = ""; if (source && Array.isArray(source) && source.length > 0) { const sourceLines = source - .map( - (line) => - `
${line - .replace(/&/g, "&") - .replace(//g, ">")}
`, - ) + .map((line) => { + const isCurrent = line.startsWith("→"); + const escaped = line.replace(/&/g, "&").replace(//g, ">"); + return `
${escaped}
`; + }) .join(""); sourceSection = ` -
-
- Source Code: -
-
- ${sourceLines} -
-
`; - } else if (source) { - // Show debug info if source exists but isn't an array - sourceSection = ` -
-
- [Debug] - Source data type: ${typeof source} -
-
- ${JSON.stringify(source, null, 2)} -
+
+
Source Code:
+
${sourceLines}
`; } - // Resolve strings for display - const funcname = resolveString(d.data.funcname) || resolveString(d.data.name); - const filename = resolveString(d.data.filename) || ""; - - // Don't show file location for special frames like and - const isSpecialFrame = filename === "~"; const fileLocationHTML = isSpecialFrame ? "" : ` -
- ${filename}${d.data.lineno ? ":" + d.data.lineno : ""} -
`; +
${filename}${d.data.lineno ? ":" + d.data.lineno : ""}
`; const tooltipHTML = ` -
-
- ${funcname} -
+
+
${funcname}
${fileLocationHTML} -
- Execution Time: - ${timeMs} ms - - Percentage: - ${percentage}% - - ${calls > 0 ? ` - Function Calls: - ${calls.toLocaleString()} - ` : ''} - - ${childCount > 0 ? ` - Child Functions: - ${childCount} - ` : ''} -
- ${sourceSection} -
- ${childCount > 0 ? - "Click to focus on this function" : - "Leaf function - no children"} -
+
+
+ Execution Time: + ${timeMs} ms + + Percentage: + ${percentage}% + + ${calls > 0 ? ` + Function Calls: + ${calls.toLocaleString()} + ` : ''} + + ${childCount > 0 ? ` + Child Functions: + ${childCount} + ` : ''} +
+ ${sourceSection} +
+ ${childCount > 0 ? "Click to zoom into this function" : "Leaf function - no children"}
`; - // Get mouse position + // Position tooltip const event = d3.event || window.event; const mouseX = event.pageX || event.clientX; const mouseY = event.pageY || event.clientY; + const padding = 12; - // Calculate tooltip dimensions (default to 320px width if not rendered yet) - let tooltipWidth = 320; - let tooltipHeight = 200; - if (this._tooltip && this._tooltip.node()) { - const node = this._tooltip - .style("opacity", 0) - .style("display", "block") - .node(); - tooltipWidth = node.offsetWidth || 320; - tooltipHeight = node.offsetHeight || 200; - this._tooltip.style("display", null); - } + this._tooltip.html(tooltipHTML); - // Calculate horizontal position: if overflow, show to the left of cursor - const padding = 10; - const rightEdge = mouseX + padding + tooltipWidth; - const viewportWidth = window.innerWidth; - let left; - if (rightEdge > viewportWidth) { + // Measure tooltip + const node = this._tooltip.style("display", "block").style("opacity", 0).node(); + const tooltipWidth = node.offsetWidth || 320; + const tooltipHeight = node.offsetHeight || 200; + + // Calculate position + let left = mouseX + padding; + let top = mouseY + padding; + + if (left + tooltipWidth > window.innerWidth) { left = mouseX - tooltipWidth - padding; - if (left < 0) left = padding; // prevent off left edge - } else { - left = mouseX + padding; + if (left < 0) left = padding; } - // Calculate vertical position: if overflow, show above cursor - const bottomEdge = mouseY + padding + tooltipHeight; - const viewportHeight = window.innerHeight; - let top; - if (bottomEdge > viewportHeight) { + if (top + tooltipHeight > window.innerHeight) { top = mouseY - tooltipHeight - padding; - if (top < 0) top = padding; // prevent off top edge - } else { - top = mouseY + padding; + if (top < 0) top = padding; } this._tooltip - .html(tooltipHTML) .style("left", left + "px") .style("top", top + "px") .transition() - .duration(200) + .duration(150) .style("opacity", 1); + + // Update status bar + updateStatusBar(d.data, data.value); }; - // Override the hide method pythonTooltip.hide = function () { if (this._tooltip) { - this._tooltip.transition().duration(200).style("opacity", 0); + this._tooltip.transition().duration(150).style("opacity", 0); } + clearStatusBar(); }; + return pythonTooltip; } +// ============================================================================ +// Flamegraph Creation +// ============================================================================ + +function ensureLibraryLoaded() { + if (typeof flamegraph === "undefined") { + console.error("d3-flame-graph library not loaded"); + document.getElementById("chart").innerHTML = + '
Error: d3-flame-graph library failed to load
'; + throw new Error("d3-flame-graph library failed to load"); + } +} + +const HEAT_THRESHOLDS = [ + [0.6, 8], + [0.35, 7], + [0.18, 6], + [0.12, 5], + [0.06, 4], + [0.03, 3], + [0.01, 2], +]; + +function getHeatLevel(percentage) { + for (const [threshold, level] of HEAT_THRESHOLDS) { + if (percentage >= threshold) return level; + } + return 1; +} + +function getHeatColors() { + const style = getComputedStyle(document.documentElement); + const colors = {}; + for (let i = 1; i <= 8; i++) { + colors[i] = style.getPropertyValue(`--heat-${i}`).trim(); + } + return colors; +} + function createFlamegraph(tooltip, rootValue) { + const chartArea = document.querySelector('.chart-area'); + const width = chartArea ? chartArea.clientWidth - 32 : window.innerWidth - 320; + const heatColors = getHeatColors(); + let chart = flamegraph() - .width(window.innerWidth - 80) + .width(width) .cellHeight(20) .transitionDuration(300) .minFrameSize(1) @@ -265,142 +381,207 @@ function createFlamegraph(tooltip, rootValue) { .inverted(true) .setColorMapper(function (d) { const percentage = d.data.value / rootValue; - let colorIndex; - if (percentage >= 0.6) colorIndex = 7; - else if (percentage >= 0.35) colorIndex = 6; - else if (percentage >= 0.18) colorIndex = 5; - else if (percentage >= 0.12) colorIndex = 4; - else if (percentage >= 0.06) colorIndex = 3; - else if (percentage >= 0.03) colorIndex = 2; - else if (percentage >= 0.01) colorIndex = 1; - else colorIndex = 0; // <1% - return pythonColors[colorIndex]; + const level = getHeatLevel(percentage); + return heatColors[level]; }); + return chart; } function renderFlamegraph(chart, data) { d3.select("#chart").datum(data).call(chart); - window.flamegraphChart = chart; // for controls - window.flamegraphData = data; // for resize/search + window.flamegraphChart = chart; + window.flamegraphData = data; populateStats(data); } -function attachPanelControls() { - const infoBtn = document.getElementById("show-info-btn"); - const infoPanel = document.getElementById("info-panel"); - const closeBtn = document.getElementById("close-info-btn"); - if (infoBtn && infoPanel) { - infoBtn.addEventListener("click", function () { - const isOpen = infoPanel.style.display === "block"; - infoPanel.style.display = isOpen ? "none" : "block"; - }); - } - if (closeBtn && infoPanel) { - closeBtn.addEventListener("click", function () { - infoPanel.style.display = "none"; - }); - } -} +// ============================================================================ +// Search +// ============================================================================ function updateSearchHighlight(searchTerm, searchInput) { d3.selectAll("#chart rect") - .style("stroke", null) - .style("stroke-width", null) - .style("opacity", null); + .classed("search-match", false) + .classed("search-dim", false); + + // Clear active state from all hotspots + document.querySelectorAll('.hotspot').forEach(h => h.classList.remove('active')); + if (searchTerm && searchTerm.length > 0) { - d3.selectAll("#chart rect").style("opacity", 0.3); let matchCount = 0; + d3.selectAll("#chart rect").each(function (d) { if (d && d.data) { const name = resolveString(d.data.name) || ""; const funcname = resolveString(d.data.funcname) || ""; const filename = resolveString(d.data.filename) || ""; + const lineno = d.data.lineno; const term = searchTerm.toLowerCase(); - const matches = - name.toLowerCase().includes(term) || - funcname.toLowerCase().includes(term) || - filename.toLowerCase().includes(term); + + // Check if search term looks like file:line pattern + const fileLineMatch = term.match(/^(.+):(\d+)$/); + let matches = false; + + if (fileLineMatch) { + // Exact file:line matching + const searchFile = fileLineMatch[1]; + const searchLine = parseInt(fileLineMatch[2], 10); + const basename = filename.split('/').pop().toLowerCase(); + matches = basename.includes(searchFile) && lineno === searchLine; + } else { + // Regular substring search + matches = + name.toLowerCase().includes(term) || + funcname.toLowerCase().includes(term) || + filename.toLowerCase().includes(term); + } + if (matches) { matchCount++; - d3.select(this) - .style("opacity", 1) - .style("stroke", "#ff6b35") - .style("stroke-width", "2px") - .style("stroke-dasharray", "3,3"); + d3.select(this).classed("search-match", true); + } else { + d3.select(this).classed("search-dim", true); } } }); + if (searchInput) { - if (matchCount > 0) { - searchInput.style.borderColor = "rgba(40, 167, 69, 0.8)"; - searchInput.style.boxShadow = "0 6px 20px rgba(40, 167, 69, 0.2)"; - } else { - searchInput.style.borderColor = "rgba(220, 53, 69, 0.8)"; - searchInput.style.boxShadow = "0 6px 20px rgba(220, 53, 69, 0.2)"; - } + searchInput.classList.remove("has-matches", "no-matches"); + searchInput.classList.add(matchCount > 0 ? "has-matches" : "no-matches"); } + + // Mark matching hotspot as active + document.querySelectorAll('.hotspot').forEach(h => { + if (h.dataset.searchterm && h.dataset.searchterm.toLowerCase() === searchTerm.toLowerCase()) { + h.classList.add('active'); + } + }); } else if (searchInput) { - searchInput.style.borderColor = "rgba(255, 255, 255, 0.2)"; - searchInput.style.boxShadow = "0 4px 12px rgba(0, 0, 0, 0.1)"; + searchInput.classList.remove("has-matches", "no-matches"); + } +} + +function searchForHotspot(funcname) { + const searchInput = document.getElementById('search-input'); + const searchWrapper = document.querySelector('.search-wrapper'); + if (searchInput) { + // Toggle: if already searching for this term, clear it + if (searchInput.value.trim() === funcname) { + clearSearch(); + } else { + searchInput.value = funcname; + if (searchWrapper) { + searchWrapper.classList.add('has-value'); + } + performSearch(); + } } } function initSearchHandlers() { const searchInput = document.getElementById("search-input"); + const searchWrapper = document.querySelector(".search-wrapper"); if (!searchInput) return; + let searchTimeout; function performSearch() { const term = searchInput.value.trim(); updateSearchHighlight(term, searchInput); + // Toggle has-value class for clear button visibility + if (searchWrapper) { + searchWrapper.classList.toggle("has-value", term.length > 0); + } } + searchInput.addEventListener("input", function () { clearTimeout(searchTimeout); searchTimeout = setTimeout(performSearch, 150); }); + window.performSearch = performSearch; } -function handleResize(chart, data) { - window.addEventListener("resize", function () { - if (chart && data) { - const newWidth = window.innerWidth - 80; - chart.width(newWidth); - d3.select("#chart").datum(data).call(chart); +function clearSearch() { + const searchInput = document.getElementById("search-input"); + const searchWrapper = document.querySelector(".search-wrapper"); + if (searchInput) { + searchInput.value = ""; + searchInput.classList.remove("has-matches", "no-matches"); + if (searchWrapper) { + searchWrapper.classList.remove("has-value"); } + // Clear highlights + d3.selectAll("#chart rect") + .classed("search-match", false) + .classed("search-dim", false); + // Clear active hotspot + document.querySelectorAll('.hotspot').forEach(h => h.classList.remove('active')); + } +} + +// ============================================================================ +// Resize Handler +// ============================================================================ + +function handleResize() { + let resizeTimeout; + window.addEventListener("resize", function () { + clearTimeout(resizeTimeout); + resizeTimeout = setTimeout(resizeChart, 100); }); } -function initFlamegraph() { - ensureLibraryLoaded(); +function initSidebarResize() { + const sidebar = document.getElementById('sidebar'); + const resizeHandle = document.getElementById('sidebar-resize-handle'); + if (!sidebar || !resizeHandle) return; + + let isResizing = false; + let startX = 0; + let startWidth = 0; + const minWidth = 200; + const maxWidth = 600; + + resizeHandle.addEventListener('mousedown', function(e) { + isResizing = true; + startX = e.clientX; + startWidth = sidebar.offsetWidth; + resizeHandle.classList.add('resizing'); + document.body.classList.add('resizing-sidebar'); + e.preventDefault(); + }); - // Extract string table if present and resolve string indices - let processedData = EMBEDDED_DATA; - if (EMBEDDED_DATA.strings) { - stringTable = EMBEDDED_DATA.strings; - processedData = resolveStringIndices(EMBEDDED_DATA); - } + document.addEventListener('mousemove', function(e) { + if (!isResizing) return; - // Store original data for filtering - originalData = processedData; + const deltaX = e.clientX - startX; + const newWidth = Math.min(Math.max(startWidth + deltaX, minWidth), maxWidth); + sidebar.style.width = newWidth + 'px'; + e.preventDefault(); + }); - // Initialize thread filter dropdown - initThreadFilter(processedData); + document.addEventListener('mouseup', function() { + if (isResizing) { + isResizing = false; + resizeHandle.classList.remove('resizing'); + document.body.classList.remove('resizing-sidebar'); - const tooltip = createPythonTooltip(processedData); - const chart = createFlamegraph(tooltip, processedData.value); - renderFlamegraph(chart, processedData); - attachPanelControls(); - initSearchHandlers(); - handleResize(chart, processedData); -} + // Save the new width + const width = sidebar.offsetWidth; + localStorage.setItem('flamegraph-sidebar-width', width); -if (document.readyState === "loading") { - document.addEventListener("DOMContentLoaded", initFlamegraph); -} else { - initFlamegraph(); + // Resize chart after sidebar resize + setTimeout(() => { + resizeChart(); + }, 10); + } + }); } +// ============================================================================ +// Thread Stats +// ============================================================================ + // Mode constants (must match constants.py) const PROFILING_MODE_WALL = 0; const PROFILING_MODE_CPU = 1; @@ -408,97 +589,164 @@ const PROFILING_MODE_GIL = 2; const PROFILING_MODE_ALL = 3; function populateThreadStats(data, selectedThreadId = null) { - // Check if thread statistics are available const stats = data?.stats; if (!stats || !stats.thread_stats) { - return; // No thread stats available + return; } const mode = stats.mode !== undefined ? stats.mode : PROFILING_MODE_WALL; let threadStats; - // If a specific thread is selected, use per-thread stats if (selectedThreadId !== null && stats.per_thread_stats && stats.per_thread_stats[selectedThreadId]) { threadStats = stats.per_thread_stats[selectedThreadId]; } else { threadStats = stats.thread_stats; } - // Validate threadStats object - if (!threadStats || typeof threadStats.total !== 'number') { - return; // Invalid thread stats + if (!threadStats || typeof threadStats.total !== 'number' || threadStats.total <= 0) { + return; } - const bar = document.getElementById('thread-stats-bar'); - if (!bar) { - return; // DOM element not found + const section = document.getElementById('thread-stats-bar'); + if (!section) { + return; } - // Show the bar if we have valid thread stats - if (threadStats.total > 0) { - bar.style.display = 'flex'; + section.style.display = 'block'; - // Hide/show GIL stats items in GIL mode - const gilHeldStat = document.getElementById('gil-held-stat'); - const gilReleasedStat = document.getElementById('gil-released-stat'); - const gilWaitingStat = document.getElementById('gil-waiting-stat'); - const separators = bar.querySelectorAll('.thread-stat-separator'); + const gilHeldStat = document.getElementById('gil-held-stat'); + const gilReleasedStat = document.getElementById('gil-released-stat'); + const gilWaitingStat = document.getElementById('gil-waiting-stat'); - if (mode === PROFILING_MODE_GIL) { - // In GIL mode, hide GIL-related stats - if (gilHeldStat) gilHeldStat.style.display = 'none'; - if (gilReleasedStat) gilReleasedStat.style.display = 'none'; - if (gilWaitingStat) gilWaitingStat.style.display = 'none'; - separators.forEach((sep, i) => { - if (i < 3) sep.style.display = 'none'; - }); - } else { - // Show all stats in other modes - if (gilHeldStat) gilHeldStat.style.display = 'inline-flex'; - if (gilReleasedStat) gilReleasedStat.style.display = 'inline-flex'; - if (gilWaitingStat) gilWaitingStat.style.display = 'inline-flex'; - separators.forEach(sep => sep.style.display = 'inline'); - - // GIL Held - const gilHeldPct = threadStats.has_gil_pct || 0; - const gilHeldPctElem = document.getElementById('gil-held-pct'); - if (gilHeldPctElem) gilHeldPctElem.textContent = `${gilHeldPct.toFixed(2)}%`; - - // GIL Released (threads running without GIL) - const gilReleasedPct = threadStats.on_cpu_pct || 0; - const gilReleasedPctElem = document.getElementById('gil-released-pct'); - if (gilReleasedPctElem) gilReleasedPctElem.textContent = `${gilReleasedPct.toFixed(2)}%`; - - // Waiting for GIL - const gilWaitingPct = threadStats.gil_requested_pct || 0; - const gilWaitingPctElem = document.getElementById('gil-waiting-pct'); - if (gilWaitingPctElem) gilWaitingPctElem.textContent = `${gilWaitingPct.toFixed(2)}%`; - } + if (mode === PROFILING_MODE_GIL) { + // In GIL mode, hide GIL-related stats + if (gilHeldStat) gilHeldStat.style.display = 'none'; + if (gilReleasedStat) gilReleasedStat.style.display = 'none'; + if (gilWaitingStat) gilWaitingStat.style.display = 'none'; + } else { + // Show all stats + if (gilHeldStat) gilHeldStat.style.display = 'block'; + if (gilReleasedStat) gilReleasedStat.style.display = 'block'; + if (gilWaitingStat) gilWaitingStat.style.display = 'block'; - // Garbage Collection (always show) - const gcPct = threadStats.gc_pct || 0; - const gcPctElem = document.getElementById('gc-pct'); - if (gcPctElem) gcPctElem.textContent = `${gcPct.toFixed(2)}%`; + const gilHeldPctElem = document.getElementById('gil-held-pct'); + if (gilHeldPctElem) gilHeldPctElem.textContent = `${(threadStats.has_gil_pct || 0).toFixed(1)}%`; + + const gilReleasedPctElem = document.getElementById('gil-released-pct'); + if (gilReleasedPctElem) gilReleasedPctElem.textContent = `${(threadStats.on_cpu_pct || 0).toFixed(1)}%`; + + const gilWaitingPctElem = document.getElementById('gil-waiting-pct'); + if (gilWaitingPctElem) gilWaitingPctElem.textContent = `${(threadStats.gil_requested_pct || 0).toFixed(1)}%`; } + + const gcPctElem = document.getElementById('gc-pct'); + if (gcPctElem) gcPctElem.textContent = `${(threadStats.gc_pct || 0).toFixed(1)}%`; } +// ============================================================================ +// Profile Summary Stats +// ============================================================================ + +function formatNumber(num) { + if (num >= 1000000) return (num / 1000000).toFixed(1) + 'M'; + if (num >= 1000) return (num / 1000).toFixed(1) + 'K'; + return num.toLocaleString(); +} + +function formatDuration(seconds) { + if (seconds >= 3600) { + const h = Math.floor(seconds / 3600); + const m = Math.floor((seconds % 3600) / 60); + return `${h}h ${m}m`; + } + if (seconds >= 60) { + const m = Math.floor(seconds / 60); + const s = Math.floor(seconds % 60); + return `${m}m ${s}s`; + } + return seconds.toFixed(2) + 's'; +} + +function populateProfileSummary(data) { + const stats = data.stats || {}; + const totalSamples = stats.total_samples || data.value || 0; + const duration = stats.duration_sec || 0; + const sampleRate = stats.sample_rate || (duration > 0 ? totalSamples / duration : 0); + const errorRate = stats.error_rate || 0; + const missedSamples= stats.missed_samples || 0; + + const samplesEl = document.getElementById('stat-total-samples'); + if (samplesEl) samplesEl.textContent = formatNumber(totalSamples); + + const durationEl = document.getElementById('stat-duration'); + if (durationEl) durationEl.textContent = duration > 0 ? formatDuration(duration) : '--'; + + const rateEl = document.getElementById('stat-sample-rate'); + if (rateEl) rateEl.textContent = sampleRate > 0 ? formatNumber(Math.round(sampleRate)) : '--'; + + // Count unique functions + let functionCount = 0; + function countFunctions(node) { + if (!node) return; + functionCount++; + if (node.children) node.children.forEach(countFunctions); + } + countFunctions(data); + + const functionsEl = document.getElementById('stat-functions'); + if (functionsEl) functionsEl.textContent = formatNumber(functionCount); + + // Efficiency bar + if (errorRate !== undefined && errorRate !== null) { + const efficiency = Math.max(0, Math.min(100, (100 - errorRate))); + + const efficiencySection = document.getElementById('efficiency-section'); + if (efficiencySection) efficiencySection.style.display = 'block'; + + const efficiencyValue = document.getElementById('stat-efficiency'); + if (efficiencyValue) efficiencyValue.textContent = efficiency.toFixed(1) + '%'; + + const efficiencyFill = document.getElementById('efficiency-fill'); + if (efficiencyFill) efficiencyFill.style.width = efficiency + '%'; + } + // MissedSamples bar + if (missedSamples !== undefined && missedSamples !== null) { + const sampleEfficiency = Math.max(0, missedSamples); + + const efficiencySection = document.getElementById('efficiency-section'); + if (efficiencySection) efficiencySection.style.display = 'block'; + + const sampleEfficiencyValue = document.getElementById('stat-missed-samples'); + if (sampleEfficiencyValue) sampleEfficiencyValue.textContent = sampleEfficiency.toFixed(1) + '%'; + + const sampleEfficiencyFill = document.getElementById('missed-samples-fill'); + if (sampleEfficiencyFill) sampleEfficiencyFill.style.width = sampleEfficiency + '%'; + } +} + +// ============================================================================ +// Hotspot Stats +// ============================================================================ + function populateStats(data) { const totalSamples = data.value || 0; + // Populate profile summary + populateProfileSummary(data); + // Populate thread statistics if available populateThreadStats(data); - // Collect all functions with their metrics, aggregated by function name const functionMap = new Map(); function collectFunctions(node) { if (!node) return; - let filename = typeof node.filename === 'number' ? resolveString(node.filename) : node.filename; - let funcname = typeof node.funcname === 'number' ? resolveString(node.funcname) : node.funcname; + let filename = resolveString(node.filename); + let funcname = resolveString(node.funcname); if (!filename || !funcname) { - const nameStr = typeof node.name === 'number' ? resolveString(node.name) : node.name; + const nameStr = resolveString(node.name); if (nameStr?.includes('(')) { const match = nameStr.match(/^(.+?)\s*\((.+?):(\d+)\)$/); if (match) { @@ -512,21 +760,18 @@ function populateStats(data) { funcname = funcname || 'unknown'; if (filename !== 'unknown' && funcname !== 'unknown' && node.value > 0) { - // Calculate direct samples (this node's value minus children's values) let childrenValue = 0; if (node.children) { childrenValue = node.children.reduce((sum, child) => sum + child.value, 0); } const directSamples = Math.max(0, node.value - childrenValue); - // Use file:line:funcname as key to ensure uniqueness const funcKey = `${filename}:${node.lineno || '?'}:${funcname}`; if (functionMap.has(funcKey)) { const existing = functionMap.get(funcKey); existing.directSamples += directSamples; existing.directPercent = (existing.directSamples / totalSamples) * 100; - // Keep the most representative file/line (the one with more samples) if (directSamples > existing.maxSingleSamples) { existing.filename = filename; existing.lineno = node.lineno || '?'; @@ -551,96 +796,81 @@ function populateStats(data) { collectFunctions(data); - // Convert map to array and get top 3 hotspots const hotSpots = Array.from(functionMap.values()) - .filter(f => f.directPercent > 0.5) // At least 0.5% to be significant + .filter(f => f.directPercent > 0.5) .sort((a, b) => b.directPercent - a.directPercent) .slice(0, 3); - // Populate the 3 cards + // Populate and animate hotspot cards for (let i = 0; i < 3; i++) { const num = i + 1; - if (i < hotSpots.length && hotSpots[i]) { - const hotspot = hotSpots[i]; - const filename = hotspot.filename || 'unknown'; - const lineno = hotspot.lineno ?? '?'; - let funcDisplay = hotspot.funcname || 'unknown'; - if (funcDisplay.length > 35) { - funcDisplay = funcDisplay.substring(0, 32) + '...'; - } + const card = document.getElementById(`hotspot-${num}`); + const funcEl = document.getElementById(`hotspot-func-${num}`); + const fileEl = document.getElementById(`hotspot-file-${num}`); + const percentEl = document.getElementById(`hotspot-percent-${num}`); + const samplesEl = document.getElementById(`hotspot-samples-${num}`); - // Don't show file:line for special frames like and + if (i < hotSpots.length && hotSpots[i]) { + const h = hotSpots[i]; + const filename = h.filename || 'unknown'; + const lineno = h.lineno ?? '?'; const isSpecialFrame = filename === '~' && (lineno === 0 || lineno === '?'); - let fileDisplay; - if (isSpecialFrame) { - fileDisplay = '--'; - } else { - const basename = filename !== 'unknown' ? filename.split('/').pop() : 'unknown'; - fileDisplay = `${basename}:${lineno}`; - } - document.getElementById(`hotspot-file-${num}`).textContent = fileDisplay; - document.getElementById(`hotspot-func-${num}`).textContent = funcDisplay; - document.getElementById(`hotspot-detail-${num}`).textContent = `${hotspot.directPercent.toFixed(1)}% samples (${hotspot.directSamples.toLocaleString()})`; + let funcDisplay = h.funcname || 'unknown'; + if (funcDisplay.length > 28) funcDisplay = funcDisplay.substring(0, 25) + '...'; + + if (funcEl) funcEl.textContent = funcDisplay; + if (fileEl) { + if (isSpecialFrame) { + fileEl.textContent = '--'; + } else { + const basename = filename !== 'unknown' ? filename.split('/').pop() : 'unknown'; + fileEl.textContent = `${basename}:${lineno}`; + } + } + if (percentEl) percentEl.textContent = `${h.directPercent.toFixed(1)}%`; + if (samplesEl) samplesEl.textContent = ` (${h.directSamples.toLocaleString()})`; } else { - document.getElementById(`hotspot-file-${num}`).textContent = '--'; - document.getElementById(`hotspot-func-${num}`).textContent = '--'; - document.getElementById(`hotspot-detail-${num}`).textContent = '--'; + if (funcEl) funcEl.textContent = '--'; + if (fileEl) fileEl.textContent = '--'; + if (percentEl) percentEl.textContent = '--'; + if (samplesEl) samplesEl.textContent = ''; } - } -} - -// Control functions -function resetZoom() { - if (window.flamegraphChart) { - window.flamegraphChart.resetZoom(); - } -} -function exportSVG() { - const svgElement = document.querySelector("#chart svg"); - if (svgElement) { - const serializer = new XMLSerializer(); - const svgString = serializer.serializeToString(svgElement); - const blob = new Blob([svgString], { type: "image/svg+xml" }); - const url = URL.createObjectURL(blob); - const a = document.createElement("a"); - a.href = url; - a.download = "python-performance-flamegraph.svg"; - a.click(); - URL.revokeObjectURL(url); - } -} - -function toggleLegend() { - const legendPanel = document.getElementById("legend-panel"); - const isHidden = - legendPanel.style.display === "none" || legendPanel.style.display === ""; - legendPanel.style.display = isHidden ? "block" : "none"; -} + // Add click handler and animate entrance + if (card) { + if (i < hotSpots.length && hotSpots[i]) { + const h = hotSpots[i]; + const basename = h.filename !== 'unknown' ? h.filename.split('/').pop() : ''; + const searchTerm = basename && h.lineno !== '?' ? `${basename}:${h.lineno}` : h.funcname; + card.dataset.searchterm = searchTerm; + card.onclick = () => searchForHotspot(searchTerm); + card.style.cursor = 'pointer'; + } else { + card.onclick = null; + delete card.dataset.searchterm; + card.style.cursor = 'default'; + } -function clearSearch() { - const searchInput = document.getElementById("search-input"); - if (searchInput) { - searchInput.value = ""; - if (window.flamegraphChart) { - window.flamegraphChart.clear(); + setTimeout(() => { + card.classList.add('visible'); + }, 100 + i * 80); } } } +// ============================================================================ +// Thread Filter +// ============================================================================ + function initThreadFilter(data) { const threadFilter = document.getElementById('thread-filter'); - const threadWrapper = document.querySelector('.thread-filter-wrapper'); + const threadSection = document.getElementById('thread-section'); - if (!threadFilter || !data.threads) { - return; - } + if (!threadFilter || !data.threads) return; - // Clear existing options except "All Threads" threadFilter.innerHTML = ''; - // Add thread options const threads = data.threads || []; threads.forEach(threadId => { const option = document.createElement('option'); @@ -649,9 +879,8 @@ function initThreadFilter(data) { threadFilter.appendChild(option); }); - // Show filter if more than one thread - if (threads.length > 1 && threadWrapper) { - threadWrapper.style.display = 'inline-flex'; + if (threads.length > 1 && threadSection) { + threadSection.style.display = 'block'; } } @@ -666,11 +895,9 @@ function filterByThread() { let selectedThreadId = null; if (selectedThread === 'all') { - // Show all data filteredData = originalData; } else { - // Filter data by thread - selectedThreadId = parseInt(selectedThread); + selectedThreadId = parseInt(selectedThread, 10); filteredData = filterDataByThread(originalData, selectedThreadId); if (filteredData.strings) { @@ -679,12 +906,10 @@ function filterByThread() { } } - // Re-render flamegraph with filtered data const tooltip = createPythonTooltip(filteredData); const chart = createFlamegraph(tooltip, filteredData.value); renderFlamegraph(chart, filteredData); - // Update thread stats to show per-thread or aggregate stats populateThreadStats(originalData, selectedThreadId); } @@ -694,10 +919,7 @@ function filterDataByThread(data, threadId) { return null; } - const filteredNode = { - ...node, - children: [] - }; + const filteredNode = { ...node, children: [] }; if (node.children && Array.isArray(node.children)) { filteredNode.children = node.children @@ -708,17 +930,6 @@ function filterDataByThread(data, threadId) { return filteredNode; } - const filteredRoot = { - ...data, - children: [] - }; - - if (data.children && Array.isArray(data.children)) { - filteredRoot.children = data.children - .map(child => filterNode(child)) - .filter(child => child !== null); - } - function recalculateValue(node) { if (!node.children || node.children.length === 0) { return node.value || 0; @@ -728,8 +939,72 @@ function filterDataByThread(data, threadId) { return node.value; } - recalculateValue(filteredRoot); + const filteredRoot = { ...data, children: [] }; + if (data.children && Array.isArray(data.children)) { + filteredRoot.children = data.children + .map(child => filterNode(child)) + .filter(child => child !== null); + } + + recalculateValue(filteredRoot); return filteredRoot; } +// ============================================================================ +// Control Functions +// ============================================================================ + +function resetZoom() { + if (window.flamegraphChart) { + window.flamegraphChart.resetZoom(); + } +} + +function exportSVG() { + const svgElement = document.querySelector("#chart svg"); + if (!svgElement) { + console.warn("Cannot export: No flamegraph SVG found"); + return; + } + const serializer = new XMLSerializer(); + const svgString = serializer.serializeToString(svgElement); + const blob = new Blob([svgString], { type: "image/svg+xml" }); + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = "python-performance-flamegraph.svg"; + a.click(); + URL.revokeObjectURL(url); +} + +// ============================================================================ +// Initialization +// ============================================================================ + +function initFlamegraph() { + ensureLibraryLoaded(); + restoreUIState(); + + let processedData = EMBEDDED_DATA; + if (EMBEDDED_DATA.strings) { + stringTable = EMBEDDED_DATA.strings; + processedData = resolveStringIndices(EMBEDDED_DATA); + } + + originalData = processedData; + initThreadFilter(processedData); + + const tooltip = createPythonTooltip(processedData); + const chart = createFlamegraph(tooltip, processedData.value); + renderFlamegraph(chart, processedData); + initSearchHandlers(); + initSidebarResize(); + handleResize(); +} + +if (document.readyState === "loading") { + document.addEventListener("DOMContentLoaded", initFlamegraph); +} else { + initFlamegraph(); +} diff --git a/Lib/profiling/sampling/flamegraph_template.html b/Lib/profiling/sampling/flamegraph_template.html index 5f94bbe69c4f4f..09b673b76da506 100644 --- a/Lib/profiling/sampling/flamegraph_template.html +++ b/Lib/profiling/sampling/flamegraph_template.html @@ -1,9 +1,9 @@ - + - Python Performance Flamegraph + Tachyon Profiler - Flamegraph @@ -11,165 +11,297 @@ -
-
- -
-

Tachyon Profiler Performance Flamegraph

-
- Interactive visualization of function call performance -
+
+ +
+
+ Tachyon + + Profiler
- -
+
+ + + +
+ - - + +
+ + + + +
+
+
-
-
-
+ +
+ + + + +
diff --git a/Lib/profiling/sampling/sample.py b/Lib/profiling/sampling/sample.py index 82c0d3959ba22d..88d9a4fa13baf9 100644 --- a/Lib/profiling/sampling/sample.py +++ b/Lib/profiling/sampling/sample.py @@ -112,8 +112,10 @@ def sample(self, collector, duration_sec=10): if self.realtime_stats and len(self.sample_intervals) > 0: print() # Add newline after real-time stats - sample_rate = num_samples / running_time + sample_rate = num_samples / running_time if running_time > 0 else 0 error_rate = (errors / num_samples) * 100 if num_samples > 0 else 0 + expected_samples = int(duration_sec / sample_interval_sec) + missed_samples = (expected_samples - num_samples) / expected_samples * 100 if expected_samples > 0 else 0 # Don't print stats for live mode (curses is handling display) is_live_mode = LiveStatsCollector is not None and isinstance(collector, LiveStatsCollector) @@ -124,9 +126,8 @@ def sample(self, collector, duration_sec=10): # Pass stats to flamegraph collector if it's the right type if hasattr(collector, 'set_stats'): - collector.set_stats(self.sample_interval_usec, running_time, sample_rate, error_rate, mode=self.mode) + collector.set_stats(self.sample_interval_usec, running_time, sample_rate, error_rate, missed_samples, mode=self.mode) - expected_samples = int(duration_sec / sample_interval_sec) if num_samples < expected_samples and not is_live_mode and not interrupted: print( f"Warning: missed {expected_samples - num_samples} samples " diff --git a/Lib/profiling/sampling/stack_collector.py b/Lib/profiling/sampling/stack_collector.py index 9028a8bebb19b4..146a058a03ac14 100644 --- a/Lib/profiling/sampling/stack_collector.py +++ b/Lib/profiling/sampling/stack_collector.py @@ -113,13 +113,15 @@ def collect(self, stack_frames, skip_idle=False): # Call parent collect to process frames super().collect(stack_frames, skip_idle=skip_idle) - def set_stats(self, sample_interval_usec, duration_sec, sample_rate, error_rate=None, mode=None): + def set_stats(self, sample_interval_usec, duration_sec, sample_rate, + error_rate=None, missed_samples=None, mode=None): """Set profiling statistics to include in flamegraph data.""" self.stats = { "sample_interval_usec": sample_interval_usec, "duration_sec": duration_sec, "sample_rate": sample_rate, "error_rate": error_rate, + "missed_samples": missed_samples, "mode": mode } diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py index 38665f5a591eec..e8c12c2221549a 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py @@ -494,7 +494,7 @@ def test_flamegraph_collector_export(self): # Should be valid HTML self.assertIn("", content.lower()) self.assertIn(" Date: Mon, 1 Dec 2025 18:37:46 +0000 Subject: [PATCH 233/638] gh-138122: Small fixes to the new tachyon UI (#142157) --- Lib/profiling/sampling/flamegraph.js | 4 +++- Lib/profiling/sampling/flamegraph_template.html | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/profiling/sampling/flamegraph.js b/Lib/profiling/sampling/flamegraph.js index 7a2b2ef2e3135e..494d156a8dddfc 100644 --- a/Lib/profiling/sampling/flamegraph.js +++ b/Lib/profiling/sampling/flamegraph.js @@ -633,7 +633,9 @@ function populateThreadStats(data, selectedThreadId = null) { if (gilHeldPctElem) gilHeldPctElem.textContent = `${(threadStats.has_gil_pct || 0).toFixed(1)}%`; const gilReleasedPctElem = document.getElementById('gil-released-pct'); - if (gilReleasedPctElem) gilReleasedPctElem.textContent = `${(threadStats.on_cpu_pct || 0).toFixed(1)}%`; + // GIL Released = not holding GIL and not waiting for it + const gilReleasedPct = Math.max(0, 100 - (threadStats.has_gil_pct || 0) - (threadStats.gil_requested_pct || 0)); + if (gilReleasedPctElem) gilReleasedPctElem.textContent = `${gilReleasedPct.toFixed(1)}%`; const gilWaitingPctElem = document.getElementById('gil-waiting-pct'); if (gilWaitingPctElem) gilWaitingPctElem.textContent = `${(threadStats.gil_requested_pct || 0).toFixed(1)}%`; diff --git a/Lib/profiling/sampling/flamegraph_template.html b/Lib/profiling/sampling/flamegraph_template.html index 09b673b76da506..82102c229e7af9 100644 --- a/Lib/profiling/sampling/flamegraph_template.html +++ b/Lib/profiling/sampling/flamegraph_template.html @@ -155,7 +155,7 @@

Runtime Stats

--
-
Waiting
+
Waiting GIL
--
From eb892868b31322d7cf271bc25923e14b1f67ae38 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 1 Dec 2025 19:04:47 -0500 Subject: [PATCH 234/638] gh-142048: Fix quadratically increasing GC delays (gh-142051) The GC for the free threaded build would get slower with each collection due to effectively double counting objects freed by the GC. --- .../2025-12-01-20-41-26.gh-issue-142048.c2YosX.rst | 2 ++ Python/gc_free_threading.c | 14 +++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-12-01-20-41-26.gh-issue-142048.c2YosX.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-12-01-20-41-26.gh-issue-142048.c2YosX.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-01-20-41-26.gh-issue-142048.c2YosX.rst new file mode 100644 index 00000000000000..1400dae13ffe32 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-01-20-41-26.gh-issue-142048.c2YosX.rst @@ -0,0 +1,2 @@ +Fix quadratically increasing garbage collection delays in free-threaded +build. diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index 1717603b947f90..e672e870db2f27 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -2210,7 +2210,19 @@ record_deallocation(PyThreadState *tstate) gc->alloc_count--; if (gc->alloc_count <= -LOCAL_ALLOC_COUNT_THRESHOLD) { GCState *gcstate = &tstate->interp->gc; - _Py_atomic_add_int(&gcstate->young.count, (int)gc->alloc_count); + int count = _Py_atomic_load_int_relaxed(&gcstate->young.count); + int new_count; + do { + if (count == 0) { + break; + } + new_count = count + (int)gc->alloc_count; + if (new_count < 0) { + new_count = 0; + } + } while (!_Py_atomic_compare_exchange_int(&gcstate->young.count, + &count, + new_count)); gc->alloc_count = 0; } } From 41728856a277749d022d99af82c8e14b85057f3c Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Mon, 1 Dec 2025 23:13:11 -0500 Subject: [PATCH 235/638] gh-142163: Only define `HAVE_THREAD_LOCAL` when `Py_BUILD_CORE` is set (#142164) --- Include/pyport.h | 10 ++++++++-- .../2025-12-01-18-17-16.gh-issue-142163.2HiX5A.rst | 2 ++ 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-12-01-18-17-16.gh-issue-142163.2HiX5A.rst diff --git a/Include/pyport.h b/Include/pyport.h index 97c0e195d19808..61e2317976eed1 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -509,9 +509,15 @@ extern "C" { # define Py_CAN_START_THREADS 1 #endif -#ifdef WITH_THREAD -// HAVE_THREAD_LOCAL is just defined here for compatibility's sake + +/* gh-142163: Some libraries rely on HAVE_THREAD_LOCAL being undefined, so + * we can only define it only when Py_BUILD_CORE is set.*/ +#ifdef Py_BUILD_CORE +// This is no longer coupled to _Py_thread_local. # define HAVE_THREAD_LOCAL 1 +#endif + +#ifdef WITH_THREAD # ifdef thread_local # define _Py_thread_local thread_local # elif __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_THREADS__) diff --git a/Misc/NEWS.d/next/C_API/2025-12-01-18-17-16.gh-issue-142163.2HiX5A.rst b/Misc/NEWS.d/next/C_API/2025-12-01-18-17-16.gh-issue-142163.2HiX5A.rst new file mode 100644 index 00000000000000..5edcfd81992c13 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-12-01-18-17-16.gh-issue-142163.2HiX5A.rst @@ -0,0 +1,2 @@ +Fix the ``HAVE_THREAD_LOCAL`` macro being defined without the +``Py_BUILD_CORE`` macro set after including :file:`Python.h`. From 5e58548ebe8f7ac8c6cb0bad775912caa4090515 Mon Sep 17 00:00:00 2001 From: LloydZ <35182391+cocolato@users.noreply.github.com> Date: Tue, 2 Dec 2025 12:41:54 +0800 Subject: [PATCH 236/638] gh-59000: Fix pdb breakpoint resolution for class methods when module not imported (#141949) --- Lib/pdb.py | 4 +++- Lib/test/test_pdb.py | 16 ++++++++++++++++ ...2025-11-25-16-00-29.gh-issue-59000.YtOyJy.rst | 1 + 3 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-25-16-00-29.gh-issue-59000.YtOyJy.rst diff --git a/Lib/pdb.py b/Lib/pdb.py index 60b713ebaf3d1a..18cee7e9ae60e1 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -1487,7 +1487,9 @@ def lineinfo(self, identifier): f = self.lookupmodule(parts[0]) if f: fname = f - item = parts[1] + item = parts[1] + else: + return failed answer = find_function(item, self.canonic(fname)) return answer or failed diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 34dfc220c7ed73..8d582742499815 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -4587,6 +4587,22 @@ def bar(): ])) self.assertIn('break in bar', stdout) + def test_issue_59000(self): + script = """ + def foo(): + pass + + class C: + def foo(self): + pass + """ + commands = """ + break C.foo + quit + """ + stdout, stderr = self.run_pdb_script(script, commands) + self.assertIn("The specified object 'C.foo' is not a function", stdout) + class ChecklineTests(unittest.TestCase): def setUp(self): diff --git a/Misc/NEWS.d/next/Library/2025-11-25-16-00-29.gh-issue-59000.YtOyJy.rst b/Misc/NEWS.d/next/Library/2025-11-25-16-00-29.gh-issue-59000.YtOyJy.rst new file mode 100644 index 00000000000000..33ab8a0659e4a2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-25-16-00-29.gh-issue-59000.YtOyJy.rst @@ -0,0 +1 @@ +Fix :mod:`pdb` breakpoint resolution for class methods when the module defining the class is not imported. From 2dc28eb8b0956f3ebc8ec9cedcb614fb7516e120 Mon Sep 17 00:00:00 2001 From: Krishna-web-hub Date: Tue, 2 Dec 2025 11:23:12 +0530 Subject: [PATCH 237/638] gh-140281: Update free threading Python HOWTO for 3.14 (gh-140566) Co-authored-by: Sam Gross --- Doc/howto/free-threading-python.rst | 57 +++++++++-------------------- 1 file changed, 17 insertions(+), 40 deletions(-) diff --git a/Doc/howto/free-threading-python.rst b/Doc/howto/free-threading-python.rst index e4df7a787a2b17..380c2be04957d5 100644 --- a/Doc/howto/free-threading-python.rst +++ b/Doc/howto/free-threading-python.rst @@ -11,9 +11,7 @@ available processing power by running threads in parallel on available CPU cores While not all software will benefit from this automatically, programs designed with threading in mind will run faster on multi-core hardware. -The free-threaded mode is working and continues to be improved, but -there is some additional overhead in single-threaded workloads compared -to the regular build. Additionally, third-party packages, in particular ones +Some third-party packages, in particular ones with an :term:`extension module`, may not be ready for use in a free-threaded build, and will re-enable the :term:`GIL`. @@ -101,63 +99,42 @@ This section describes known limitations of the free-threaded CPython build. Immortalization --------------- -The free-threaded build of the 3.13 release makes some objects :term:`immortal`. +In the free-threaded build, some objects are :term:`immortal`. Immortal objects are not deallocated and have reference counts that are never modified. This is done to avoid reference count contention that would prevent efficient multi-threaded scaling. -An object will be made immortal when a new thread is started for the first time -after the main thread is running. The following objects are immortalized: +As of the 3.14 release, immortalization is limited to: -* :ref:`function ` objects declared at the module level -* :ref:`method ` descriptors -* :ref:`code ` objects -* :term:`module` objects and their dictionaries -* :ref:`classes ` (type objects) - -Because immortal objects are never deallocated, applications that create many -objects of these types may see increased memory usage under Python 3.13. This -has been addressed in the 3.14 release, where the aforementioned objects use -deferred reference counting to avoid reference count contention. - -Additionally, numeric and string literals in the code as well as strings -returned by :func:`sys.intern` are also immortalized in the 3.13 release. This -behavior is part of the 3.14 release as well and it is expected to remain in -future free-threaded builds. +* Code constants: numeric literals, string literals, and tuple literals + composed of other constants. +* Strings interned by :func:`sys.intern`. Frame objects ------------- -It is not safe to access :ref:`frame ` objects from other -threads and doing so may cause your program to crash . This means that -:func:`sys._current_frames` is generally not safe to use in a free-threaded -build. Functions like :func:`inspect.currentframe` and :func:`sys._getframe` -are generally safe as long as the resulting frame object is not passed to -another thread. +It is not safe to access :attr:`frame.f_locals` from a :ref:`frame ` +object if that frame is currently executing in another thread, and doing so may +crash the interpreter. + Iterators --------- -Sharing the same iterator object between multiple threads is generally not -safe and threads may see duplicate or missing elements when iterating or crash -the interpreter. +It is generally not thread-safe to access the same iterator object from +multiple threads concurrently, and threads may see duplicate or missing +elements. Single-threaded performance --------------------------- The free-threaded build has additional overhead when executing Python code -compared to the default GIL-enabled build. In 3.13, this overhead is about -40% on the `pyperformance `_ suite. -Programs that spend most of their time in C extensions or I/O will see -less of an impact. The largest impact is because the specializing adaptive -interpreter (:pep:`659`) is disabled in the free-threaded build. - -The specializing adaptive interpreter has been re-enabled in a thread-safe way -in the 3.14 release. The performance penalty on single-threaded code in -free-threaded mode is now roughly 5-10%, depending on the platform and C -compiler used. +compared to the default GIL-enabled build. The amount of overhead depends +on the workload and hardware. On the pyperformance benchmark suite, the +average overhead ranges from about 1% on macOS aarch64 to 8% on x86-64 Linux +systems. Behavioral changes From fddc24e4c85467f14075e94fd0d23d928ceb535f Mon Sep 17 00:00:00 2001 From: LloydZ <35182391+cocolato@users.noreply.github.com> Date: Tue, 2 Dec 2025 15:40:02 +0800 Subject: [PATCH 238/638] gh-141982: Fix pdb can't set breakpoints on async functions (#141983) Co-authored-by: Tian Gao --- Lib/pdb.py | 2 +- Lib/test/test_pdb.py | 19 +++++++++++++++++++ ...-11-30-04-28-30.gh-issue-141982.pxZct9.rst | 1 + 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-30-04-28-30.gh-issue-141982.pxZct9.rst diff --git a/Lib/pdb.py b/Lib/pdb.py index 18cee7e9ae60e1..1506e3d4709817 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -130,7 +130,7 @@ def find_first_executable_line(code): return code.co_firstlineno def find_function(funcname, filename): - cre = re.compile(r'def\s+%s(\s*\[.+\])?\s*[(]' % re.escape(funcname)) + cre = re.compile(r'(?:async\s+)?def\s+%s(\s*\[.+\])?\s*[(]' % re.escape(funcname)) try: fp = tokenize.open(filename) except OSError: diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 8d582742499815..c097808e7fdc7c 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -4587,6 +4587,25 @@ def bar(): ])) self.assertIn('break in bar', stdout) + @unittest.skipIf(SKIP_CORO_TESTS, "Coroutine tests are skipped") + def test_async_break(self): + script = """ + import asyncio + + async def main(): + pass + + asyncio.run(main()) + """ + commands = """ + break main + continue + quit + """ + stdout, stderr = self.run_pdb_script(script, commands) + self.assertRegex(stdout, r"Breakpoint 1 at .*main\.py:5") + self.assertIn("pass", stdout) + def test_issue_59000(self): script = """ def foo(): diff --git a/Misc/NEWS.d/next/Library/2025-11-30-04-28-30.gh-issue-141982.pxZct9.rst b/Misc/NEWS.d/next/Library/2025-11-30-04-28-30.gh-issue-141982.pxZct9.rst new file mode 100644 index 00000000000000..e5ec593dd6e65d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-30-04-28-30.gh-issue-141982.pxZct9.rst @@ -0,0 +1 @@ +Allow :mod:`pdb` to set breakpoints on async functions with function names. From 748c4b47b70f89cb132f865250eb6c951a177366 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 2 Dec 2025 09:57:09 +0100 Subject: [PATCH 239/638] Document None for timeout argument of select.select (#142177) --- Doc/library/select.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/select.rst b/Doc/library/select.rst index e821cb01d941b2..62b5161fb80634 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -115,7 +115,7 @@ The module defines the following: :ref:`kevent-objects` below for the methods supported by kevent objects. -.. function:: select(rlist, wlist, xlist[, timeout]) +.. function:: select(rlist, wlist, xlist, timeout=None) This is a straightforward interface to the Unix :c:func:`!select` system call. The first three arguments are iterables of 'waitable objects': either @@ -131,7 +131,7 @@ The module defines the following: platform-dependent. (It is known to work on Unix but not on Windows.) The optional *timeout* argument specifies a time-out in seconds; it may be a non-integer to specify fractions of seconds. - When the *timeout* argument is omitted the function blocks until + When the *timeout* argument is omitted or ``None``, the function blocks until at least one file descriptor is ready. A time-out value of zero specifies a poll and never blocks. From d3c888b4ec15dbd7d6b6ef4f15b558af77c228af Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Tue, 2 Dec 2025 13:12:20 +0000 Subject: [PATCH 240/638] gh-139707: Fix example for configure option (GH-142153) Fix example for nre configure option --- Doc/using/configure.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index e140ca5d71f555..dff0fe036ea53a 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -339,7 +339,7 @@ General Options .. code-block:: json { - "_gdbm": "The '_gdbm' module is not available in this distribution" + "_gdbm": "The '_gdbm' module is not available in this distribution", "tkinter": "Install the python-tk package to use tkinter", "_tkinter": "Install the python-tk package to use tkinter", } From 8801c6dec79275e9b588ae03e356d8e7656fa3f0 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Tue, 2 Dec 2025 20:33:40 +0000 Subject: [PATCH 241/638] gh-140677 Add heatmap visualization to Tachyon sampling profiler (#140680) Co-authored-by: Ivona Stojanovic --- Lib/profiling/sampling/__init__.py | 3 +- Lib/profiling/sampling/_css_utils.py | 22 + .../{ => _flamegraph_assets}/flamegraph.css | 326 +---- .../{ => _flamegraph_assets}/flamegraph.js | 0 .../flamegraph_template.html | 0 .../sampling/_heatmap_assets/heatmap.css | 1146 +++++++++++++++++ .../sampling/_heatmap_assets/heatmap.js | 349 +++++ .../sampling/_heatmap_assets/heatmap_index.js | 111 ++ .../heatmap_index_template.html | 118 ++ .../heatmap_pyfile_template.html | 96 ++ .../sampling/_shared_assets/base.css | 369 ++++++ Lib/profiling/sampling/cli.py | 17 +- Lib/profiling/sampling/heatmap_collector.py | 1039 +++++++++++++++ Lib/profiling/sampling/sample.py | 2 +- Lib/profiling/sampling/stack_collector.py | 7 +- Lib/test/test_profiling/test_heatmap.py | 653 ++++++++++ Makefile.pre.in | 3 + ...-10-27-17-00-11.gh-issue-140677.hM9pTq.rst | 4 + 18 files changed, 3939 insertions(+), 326 deletions(-) create mode 100644 Lib/profiling/sampling/_css_utils.py rename Lib/profiling/sampling/{ => _flamegraph_assets}/flamegraph.css (72%) rename Lib/profiling/sampling/{ => _flamegraph_assets}/flamegraph.js (100%) rename Lib/profiling/sampling/{ => _flamegraph_assets}/flamegraph_template.html (100%) create mode 100644 Lib/profiling/sampling/_heatmap_assets/heatmap.css create mode 100644 Lib/profiling/sampling/_heatmap_assets/heatmap.js create mode 100644 Lib/profiling/sampling/_heatmap_assets/heatmap_index.js create mode 100644 Lib/profiling/sampling/_heatmap_assets/heatmap_index_template.html create mode 100644 Lib/profiling/sampling/_heatmap_assets/heatmap_pyfile_template.html create mode 100644 Lib/profiling/sampling/_shared_assets/base.css create mode 100644 Lib/profiling/sampling/heatmap_collector.py create mode 100644 Lib/test/test_profiling/test_heatmap.py create mode 100644 Misc/NEWS.d/next/Library/2025-10-27-17-00-11.gh-issue-140677.hM9pTq.rst diff --git a/Lib/profiling/sampling/__init__.py b/Lib/profiling/sampling/__init__.py index b493c6aa7eb06d..6a0bb5e5c2f387 100644 --- a/Lib/profiling/sampling/__init__.py +++ b/Lib/profiling/sampling/__init__.py @@ -7,7 +7,8 @@ from .collector import Collector from .pstats_collector import PstatsCollector from .stack_collector import CollapsedStackCollector +from .heatmap_collector import HeatmapCollector from .gecko_collector import GeckoCollector from .string_table import StringTable -__all__ = ("Collector", "PstatsCollector", "CollapsedStackCollector", "GeckoCollector", "StringTable") +__all__ = ("Collector", "PstatsCollector", "CollapsedStackCollector", "HeatmapCollector", "GeckoCollector", "StringTable") diff --git a/Lib/profiling/sampling/_css_utils.py b/Lib/profiling/sampling/_css_utils.py new file mode 100644 index 00000000000000..40912e9b3528e3 --- /dev/null +++ b/Lib/profiling/sampling/_css_utils.py @@ -0,0 +1,22 @@ +import importlib.resources + + +def get_combined_css(component: str) -> str: + template_dir = importlib.resources.files(__package__) + + base_css = (template_dir / "_shared_assets" / "base.css").read_text(encoding="utf-8") + + if component == "flamegraph": + component_css = ( + template_dir / "_flamegraph_assets" / "flamegraph.css" + ).read_text(encoding="utf-8") + elif component == "heatmap": + component_css = (template_dir / "_heatmap_assets" / "heatmap.css").read_text( + encoding="utf-8" + ) + else: + raise ValueError( + f"Unknown component: {component}. Expected 'flamegraph' or 'heatmap'." + ) + + return f"{base_css}\n\n{component_css}" diff --git a/Lib/profiling/sampling/flamegraph.css b/Lib/profiling/sampling/_flamegraph_assets/flamegraph.css similarity index 72% rename from Lib/profiling/sampling/flamegraph.css rename to Lib/profiling/sampling/_flamegraph_assets/flamegraph.css index 1703815acd9e1d..c75f2324b6d499 100644 --- a/Lib/profiling/sampling/flamegraph.css +++ b/Lib/profiling/sampling/_flamegraph_assets/flamegraph.css @@ -1,136 +1,20 @@ /* ========================================================================== - Flamegraph Viewer - CSS - Python-branded profiler with dark/light theme support - ========================================================================== */ + Flamegraph Viewer - Component-Specific CSS -/* -------------------------------------------------------------------------- - CSS Variables & Theme System - -------------------------------------------------------------------------- */ - -:root { - /* Typography */ - --font-sans: "Source Sans Pro", "Lucida Grande", "Lucida Sans Unicode", - "Geneva", "Verdana", sans-serif; - --font-mono: 'SF Mono', 'Monaco', 'Consolas', 'Liberation Mono', monospace; - - /* Python brand colors (theme-independent) */ - --python-blue: #3776ab; - --python-blue-light: #4584bb; - --python-blue-lighter: #5592cc; - --python-gold: #ffd43b; - --python-gold-dark: #ffcd02; - --python-gold-light: #ffdc5c; - - /* Heat palette - defined per theme below */ - - /* Layout */ - --sidebar-width: 280px; - --sidebar-collapsed: 44px; - --topbar-height: 52px; - --statusbar-height: 32px; - - /* Transitions */ - --transition-fast: 0.15s ease; - --transition-normal: 0.25s ease; -} - -/* Light theme (default) - Python yellow-to-blue heat palette */ -:root, [data-theme="light"] { - --bg-primary: #ffffff; - --bg-secondary: #f8f9fa; - --bg-tertiary: #e9ecef; - --border: #e9ecef; - --border-subtle: #f0f2f5; - - --text-primary: #2e3338; - --text-secondary: #5a6c7d; - --text-muted: #8b949e; - - --accent: #3776ab; - --accent-hover: #2d5aa0; - --accent-glow: rgba(55, 118, 171, 0.15); - - --shadow-sm: 0 1px 3px rgba(0, 0, 0, 0.08); - --shadow-md: 0 4px 12px rgba(0, 0, 0, 0.1); - --shadow-lg: 0 8px 24px rgba(0, 0, 0, 0.15); - - --header-gradient: linear-gradient(135deg, #3776ab 0%, #4584bb 100%); - - /* Light mode heat palette - blue to yellow to orange to red (cold to hot) */ - --heat-1: #d6e9f8; - --heat-2: #a8d0ef; - --heat-3: #7ba3d1; - --heat-4: #ffe6a8; - --heat-5: #ffd43b; - --heat-6: #ffb84d; - --heat-7: #ff9966; - --heat-8: #ff6347; -} - -/* Dark theme - teal-to-orange heat palette */ -[data-theme="dark"] { - --bg-primary: #0d1117; - --bg-secondary: #161b22; - --bg-tertiary: #21262d; - --border: #30363d; - --border-subtle: #21262d; - - --text-primary: #e6edf3; - --text-secondary: #8b949e; - --text-muted: #6e7681; - - --accent: #58a6ff; - --accent-hover: #79b8ff; - --accent-glow: rgba(88, 166, 255, 0.15); - - --shadow-sm: 0 1px 3px rgba(0, 0, 0, 0.3); - --shadow-md: 0 4px 12px rgba(0, 0, 0, 0.4); - --shadow-lg: 0 8px 24px rgba(0, 0, 0, 0.5); - - --header-gradient: linear-gradient(135deg, #21262d 0%, #30363d 100%); - - /* Dark mode heat palette - dark blue to teal to yellow to orange (cold to hot) */ - --heat-1: #1e3a5f; - --heat-2: #2d5580; - --heat-3: #4a7ba7; - --heat-4: #5a9fa8; - --heat-5: #7ec488; - --heat-6: #c4de6a; - --heat-7: #f4d44d; - --heat-8: #ff6b35; -} + DEPENDENCY: Requires _shared_assets/base.css to be loaded first + This file extends the shared foundation with flamegraph-specific styles. + ========================================================================== */ /* -------------------------------------------------------------------------- - Base Styles + Layout Overrides (Flamegraph-specific) -------------------------------------------------------------------------- */ -*, *::before, *::after { - box-sizing: border-box; -} - html, body { - margin: 0; - padding: 0; height: 100%; overflow: hidden; } -body { - font-family: var(--font-sans); - font-size: 14px; - line-height: 1.6; - color: var(--text-primary); - background: var(--bg-primary); - transition: background var(--transition-normal), color var(--transition-normal); -} - -/* -------------------------------------------------------------------------- - Layout Structure - -------------------------------------------------------------------------- */ - .app-layout { - display: flex; - flex-direction: column; height: 100vh; } @@ -141,78 +25,9 @@ body { } /* -------------------------------------------------------------------------- - Top Bar + Search Input (Flamegraph-specific) -------------------------------------------------------------------------- */ -.top-bar { - height: 56px; - background: var(--header-gradient); - display: flex; - align-items: center; - padding: 0 16px; - gap: 16px; - flex-shrink: 0; - box-shadow: 0 2px 10px rgba(55, 118, 171, 0.25); - border-bottom: 2px solid var(--python-gold); -} - -/* Brand / Logo */ -.brand { - display: flex; - align-items: center; - gap: 12px; - color: white; - text-decoration: none; - flex-shrink: 0; -} - -.brand-logo { - display: flex; - align-items: center; - justify-content: center; - width: 28px; - height: 28px; - flex-shrink: 0; -} - -/* Style the inlined SVG/img inside brand-logo */ -.brand-logo svg, -.brand-logo img { - width: 28px; - height: 28px; - display: block; - object-fit: contain; - filter: drop-shadow(0 1px 2px rgba(0, 0, 0, 0.2)); -} - -.brand-info { - display: flex; - flex-direction: column; - line-height: 1.15; -} - -.brand-text { - font-weight: 700; - font-size: 16px; - letter-spacing: -0.3px; - text-shadow: 0 1px 2px rgba(0, 0, 0, 0.15); -} - -.brand-subtitle { - font-weight: 500; - font-size: 10px; - opacity: 0.9; - text-transform: uppercase; - letter-spacing: 0.5px; -} - -.brand-divider { - width: 1px; - height: 16px; - background: rgba(255, 255, 255, 0.3); -} - -/* Search */ .search-wrapper { flex: 1; max-width: 360px; @@ -308,39 +123,6 @@ body { display: flex; } -/* Toolbar */ -.toolbar { - display: flex; - align-items: center; - gap: 6px; - margin-left: auto; -} - -.toolbar-btn { - display: flex; - align-items: center; - justify-content: center; - width: 32px; - height: 32px; - padding: 0; - font-size: 15px; - color: white; - background: rgba(255, 255, 255, 0.12); - border: 1px solid rgba(255, 255, 255, 0.18); - border-radius: 6px; - cursor: pointer; - transition: all var(--transition-fast); -} - -.toolbar-btn:hover { - background: rgba(255, 255, 255, 0.22); - border-color: rgba(255, 255, 255, 0.35); -} - -.toolbar-btn:active { - transform: scale(0.95); -} - /* -------------------------------------------------------------------------- Sidebar -------------------------------------------------------------------------- */ @@ -667,11 +449,6 @@ body.resizing-sidebar { animation: shimmer 2s ease-in-out infinite; } -@keyframes shimmer { - 0% { left: -100%; } - 100% { left: 100%; } -} - /* -------------------------------------------------------------------------- Thread Stats Grid (in Sidebar) -------------------------------------------------------------------------- */ @@ -974,56 +751,6 @@ body.resizing-sidebar { opacity: 0.25; } -/* -------------------------------------------------------------------------- - Status Bar - -------------------------------------------------------------------------- */ - -.status-bar { - height: var(--statusbar-height); - background: var(--bg-secondary); - border-top: 1px solid var(--border); - display: flex; - align-items: center; - padding: 0 16px; - gap: 16px; - font-family: var(--font-mono); - font-size: 11px; - color: var(--text-secondary); - flex-shrink: 0; -} - -.status-item { - display: flex; - align-items: center; - gap: 5px; -} - -.status-item::before { - content: ''; - width: 4px; - height: 4px; - background: var(--python-gold); - border-radius: 50%; -} - -.status-item:first-child::before { - display: none; -} - -.status-label { - color: var(--text-muted); -} - -.status-value { - color: var(--text-primary); - font-weight: 500; -} - -.status-value.accent { - color: var(--accent); - font-weight: 600; -} - /* -------------------------------------------------------------------------- Tooltip -------------------------------------------------------------------------- */ @@ -1137,38 +864,7 @@ body.resizing-sidebar { } /* -------------------------------------------------------------------------- - Animations - -------------------------------------------------------------------------- */ - -@keyframes fadeIn { - from { opacity: 0; } - to { opacity: 1; } -} - -@keyframes slideUp { - from { - opacity: 0; - transform: translateY(12px); - } - to { - opacity: 1; - transform: translateY(0); - } -} - -/* -------------------------------------------------------------------------- - Focus States (Accessibility) - -------------------------------------------------------------------------- */ - -button:focus-visible, -select:focus-visible, -input:focus-visible { - outline: 2px solid var(--python-gold); - outline-offset: 2px; -} - -/* -------------------------------------------------------------------------- - Responsive + Responsive (Flamegraph-specific) -------------------------------------------------------------------------- */ @media (max-width: 900px) { @@ -1185,20 +881,12 @@ input:focus-visible { width: var(--sidebar-collapsed); } - .brand-subtitle { - display: none; - } - .search-wrapper { max-width: 220px; } } @media (max-width: 600px) { - .toolbar-btn:not(.theme-toggle) { - display: none; - } - .search-wrapper { max-width: 160px; } diff --git a/Lib/profiling/sampling/flamegraph.js b/Lib/profiling/sampling/_flamegraph_assets/flamegraph.js similarity index 100% rename from Lib/profiling/sampling/flamegraph.js rename to Lib/profiling/sampling/_flamegraph_assets/flamegraph.js diff --git a/Lib/profiling/sampling/flamegraph_template.html b/Lib/profiling/sampling/_flamegraph_assets/flamegraph_template.html similarity index 100% rename from Lib/profiling/sampling/flamegraph_template.html rename to Lib/profiling/sampling/_flamegraph_assets/flamegraph_template.html diff --git a/Lib/profiling/sampling/_heatmap_assets/heatmap.css b/Lib/profiling/sampling/_heatmap_assets/heatmap.css new file mode 100644 index 00000000000000..44915b2a2da7b8 --- /dev/null +++ b/Lib/profiling/sampling/_heatmap_assets/heatmap.css @@ -0,0 +1,1146 @@ +/* ========================================================================== + Heatmap Viewer - Component-Specific CSS + + DEPENDENCY: Requires _shared_assets/base.css to be loaded first + This file extends the shared foundation with heatmap-specific styles. + ========================================================================== */ + +/* -------------------------------------------------------------------------- + Layout Overrides (Heatmap-specific) + -------------------------------------------------------------------------- */ + +.app-layout { + min-height: 100vh; +} + +/* Sticky top bar for heatmap views */ +.top-bar { + position: sticky; + top: 0; + z-index: 100; +} + +/* Back link in toolbar */ +.back-link { + color: white; + text-decoration: none; + padding: 6px 14px; + background: rgba(255, 255, 255, 0.12); + border: 1px solid rgba(255, 255, 255, 0.18); + border-radius: 6px; + font-size: 13px; + font-weight: 500; + transition: all var(--transition-fast); +} + +.back-link:hover { + background: rgba(255, 255, 255, 0.22); + border-color: rgba(255, 255, 255, 0.35); +} + +/* -------------------------------------------------------------------------- + Main Content Area + -------------------------------------------------------------------------- */ + +.main-content { + flex: 1; + padding: 24px 3%; + width: 100%; + max-width: 100%; +} + +/* -------------------------------------------------------------------------- + Stats Summary Cards - Enhanced with Icons & Animations + -------------------------------------------------------------------------- */ + +.stats-summary { + display: grid; + grid-template-columns: repeat(3, 1fr); + gap: 12px; + margin-bottom: 24px; +} + +.stat-card { + display: flex; + align-items: center; + gap: 12px; + background: var(--bg-primary); + border: 2px solid var(--border); + border-radius: 10px; + padding: 14px 16px; + transition: all var(--transition-fast); + animation: slideUp 0.5s ease-out backwards; + animation-delay: calc(var(--i, 0) * 0.08s); + position: relative; + overflow: hidden; +} + +.stat-card::before { + content: ''; + position: absolute; + top: 0; + left: 0; + right: 0; + height: 3px; + background: linear-gradient(90deg, var(--python-blue), var(--python-gold)); + opacity: 0; + transition: opacity var(--transition-fast); +} + +.stat-card:nth-child(1) { --i: 0; --card-color: 55, 118, 171; } +.stat-card:nth-child(2) { --i: 1; --card-color: 40, 167, 69; } +.stat-card:nth-child(3) { --i: 2; --card-color: 255, 193, 7; } +.stat-card:nth-child(4) { --i: 3; --card-color: 111, 66, 193; } +.stat-card:nth-child(5) { --i: 4; --card-color: 220, 53, 69; } +.stat-card:nth-child(6) { --i: 5; --card-color: 23, 162, 184; } + +.stat-card:hover { + border-color: rgba(var(--card-color), 0.6); + background: linear-gradient(135deg, rgba(var(--card-color), 0.08) 0%, var(--bg-primary) 100%); + transform: translateY(-2px); + box-shadow: 0 4px 16px rgba(var(--card-color), 0.15); +} + +.stat-card:hover::before { + opacity: 1; +} + +.stat-icon { + width: 40px; + height: 40px; + display: flex; + align-items: center; + justify-content: center; + font-size: 18px; + background: linear-gradient(135deg, rgba(var(--card-color), 0.15) 0%, rgba(var(--card-color), 0.05) 100%); + border: 1px solid rgba(var(--card-color), 0.2); + border-radius: 10px; + flex-shrink: 0; + transition: all var(--transition-fast); +} + +.stat-card:hover .stat-icon { + transform: scale(1.05) rotate(-2deg); + background: linear-gradient(135deg, rgba(var(--card-color), 0.25) 0%, rgba(var(--card-color), 0.1) 100%); +} + +.stat-data { + flex: 1; + min-width: 0; +} + +.stat-value { + font-family: var(--font-mono); + font-size: 1.35em; + font-weight: 800; + color: rgb(var(--card-color)); + display: block; + line-height: 1.1; + letter-spacing: -0.3px; +} + +.stat-label { + font-size: 10px; + font-weight: 600; + color: var(--text-muted); + text-transform: uppercase; + letter-spacing: 0.3px; + margin-top: 2px; +} + +/* Sparkline decoration for stats */ +.stat-sparkline { + position: absolute; + bottom: 0; + left: 0; + right: 0; + height: 30px; + opacity: 0.1; + background: linear-gradient(180deg, + transparent 0%, + rgba(var(--card-color), 0.3) 100% + ); + pointer-events: none; +} + +/* -------------------------------------------------------------------------- + Rate Cards (Error Rate, Missed Samples) with Progress Bars + -------------------------------------------------------------------------- */ + +.rate-card { + display: flex; + flex-direction: column; + gap: 12px; + background: var(--bg-primary); + border: 2px solid var(--border); + border-radius: 12px; + padding: 18px 20px; + transition: all var(--transition-fast); + animation: slideUp 0.5s ease-out backwards; + position: relative; + overflow: hidden; +} + +.rate-card:nth-child(5) { animation-delay: 0.32s; --rate-color: 220, 53, 69; } +.rate-card:nth-child(6) { animation-delay: 0.40s; --rate-color: 255, 152, 0; } + +.rate-card:hover { + border-color: rgba(var(--rate-color), 0.5); + transform: translateY(-2px); + box-shadow: 0 6px 20px rgba(var(--rate-color), 0.15); +} + +.rate-header { + display: flex; + justify-content: space-between; + align-items: center; +} + +.rate-info { + display: flex; + align-items: center; + gap: 10px; +} + +.rate-icon { + width: 36px; + height: 36px; + display: flex; + align-items: center; + justify-content: center; + font-size: 18px; + background: linear-gradient(135deg, rgba(var(--rate-color), 0.15) 0%, rgba(var(--rate-color), 0.05) 100%); + border: 1px solid rgba(var(--rate-color), 0.2); + border-radius: 10px; + flex-shrink: 0; +} + +.rate-label { + font-size: 12px; + font-weight: 600; + color: var(--text-secondary); + text-transform: uppercase; + letter-spacing: 0.3px; +} + +.rate-value { + font-family: var(--font-mono); + font-size: 1.4em; + font-weight: 800; + color: rgb(var(--rate-color)); +} + +.rate-bar { + height: 8px; + background: var(--bg-tertiary); + border-radius: 4px; + overflow: hidden; + position: relative; +} + +.rate-fill { + height: 100%; + border-radius: 4px; + transition: width 0.8s ease-out; + position: relative; + overflow: hidden; +} + +.rate-fill.error { + background: linear-gradient(90deg, #dc3545 0%, #ff6b6b 100%); +} + +.rate-fill.warning { + background: linear-gradient(90deg, #ff9800 0%, #ffc107 100%); +} + +.rate-fill.good { + background: linear-gradient(90deg, #28a745 0%, #20c997 100%); +} + +/* Shimmer animation on rate bars */ +.rate-fill::after { + content: ''; + position: absolute; + top: 0; + left: -100%; + width: 100%; + height: 100%; + background: linear-gradient( + 90deg, + transparent 0%, + rgba(255, 255, 255, 0.4) 50%, + transparent 100% + ); + animation: shimmer 2.5s ease-in-out infinite; +} + +/* -------------------------------------------------------------------------- + Section Headers + -------------------------------------------------------------------------- */ + +.section-header { + display: flex; + align-items: center; + gap: 12px; + margin-bottom: 16px; + padding-bottom: 12px; + border-bottom: 2px solid var(--python-gold); +} + +.section-title { + font-size: 18px; + font-weight: 700; + color: var(--text-primary); + margin: 0; + flex: 1; +} + +/* -------------------------------------------------------------------------- + Filter Controls + -------------------------------------------------------------------------- */ + +.filter-controls { + display: flex; + gap: 8px; + flex-wrap: wrap; + align-items: center; + margin-bottom: 16px; +} + +.control-btn { + padding: 8px 16px; + background: var(--bg-secondary); + color: var(--text-primary); + border: 1px solid var(--border); + border-radius: 6px; + font-size: 13px; + font-weight: 500; + cursor: pointer; + transition: all var(--transition-fast); +} + +.control-btn:hover { + background: var(--accent); + color: white; + border-color: var(--accent); +} + +/* -------------------------------------------------------------------------- + Type Sections (stdlib, project, etc) + -------------------------------------------------------------------------- */ + +.type-section { + background: var(--bg-primary); + border: 1px solid var(--border); + border-radius: 8px; + overflow: hidden; + margin-bottom: 12px; +} + +.type-header { + padding: 12px 16px; + background: var(--header-gradient); + color: white; + cursor: pointer; + display: flex; + align-items: center; + gap: 10px; + user-select: none; + transition: all var(--transition-fast); + font-weight: 600; +} + +.type-header:hover { + opacity: 0.95; +} + +.type-icon { + font-size: 12px; + transition: transform var(--transition-fast); + min-width: 12px; +} + +.type-title { + font-size: 14px; + flex: 1; +} + +.type-stats { + font-size: 12px; + opacity: 0.9; + background: rgba(255, 255, 255, 0.15); + padding: 4px 10px; + border-radius: 4px; + font-family: var(--font-mono); +} + +.type-content { + padding: 12px; +} + +/* -------------------------------------------------------------------------- + Folder Nodes (hierarchical structure) + -------------------------------------------------------------------------- */ + +.folder-node { + margin-bottom: 6px; +} + +.folder-header { + padding: 8px 12px; + background: var(--bg-secondary); + border: 1px solid var(--border); + border-radius: 6px; + cursor: pointer; + display: flex; + align-items: center; + gap: 8px; + user-select: none; + transition: all var(--transition-fast); +} + +.folder-header:hover { + background: var(--accent-glow); + border-color: var(--accent); +} + +.folder-icon { + font-size: 10px; + color: var(--accent); + transition: transform var(--transition-fast); + min-width: 12px; +} + +.folder-name { + flex: 1; + font-weight: 500; + color: var(--text-primary); + font-size: 13px; +} + +.folder-stats { + font-size: 11px; + color: var(--text-secondary); + background: var(--bg-tertiary); + padding: 2px 8px; + border-radius: 4px; + font-family: var(--font-mono); +} + +.folder-content { + padding-left: 20px; + margin-top: 6px; +} + +/* -------------------------------------------------------------------------- + File Items + -------------------------------------------------------------------------- */ + +.files-list { + display: flex; + flex-direction: column; + gap: 4px; + margin-top: 8px; +} + +.file-item { + display: flex; + align-items: center; + gap: 12px; + padding: 8px 12px; + background: var(--bg-primary); + border: 1px solid var(--border-subtle); + border-radius: 6px; + transition: all var(--transition-fast); +} + +.file-item:hover { + background: var(--bg-secondary); + border-color: var(--border); +} + +.file-item .file-link { + flex: 1; + min-width: 0; + font-size: 13px; +} + +.file-samples { + font-size: 12px; + color: var(--text-secondary); + font-weight: 600; + white-space: nowrap; + width: 130px; + flex-shrink: 0; + text-align: right; + font-family: var(--font-mono); +} + +.heatmap-bar-container { + width: 120px; + flex-shrink: 0; + display: flex; + align-items: center; +} + +.heatmap-bar { + flex-shrink: 0; + border-radius: 2px; +} + +/* Links */ +.file-link { + color: var(--accent); + text-decoration: none; + font-weight: 500; + transition: color var(--transition-fast); +} + +.file-link:hover { + color: var(--accent-hover); + text-decoration: underline; +} + +/* -------------------------------------------------------------------------- + Module Badges + -------------------------------------------------------------------------- */ + +.module-badge { + display: inline-block; + padding: 3px 8px; + border-radius: 4px; + font-size: 11px; + font-weight: 600; +} + +.badge-stdlib { + background: rgba(40, 167, 69, 0.15); + color: #28a745; +} + +.badge-site-packages { + background: rgba(0, 123, 255, 0.15); + color: #007bff; +} + +.badge-project { + background: rgba(255, 193, 7, 0.2); + color: #d39e00; +} + +.badge-other { + background: var(--bg-tertiary); + color: var(--text-secondary); +} + +[data-theme="dark"] .badge-stdlib { + background: rgba(40, 167, 69, 0.25); + color: #5dd879; +} + +[data-theme="dark"] .badge-site-packages { + background: rgba(88, 166, 255, 0.25); + color: #79b8ff; +} + +[data-theme="dark"] .badge-project { + background: rgba(255, 212, 59, 0.25); + color: #ffd43b; +} + +/* ========================================================================== + FILE VIEW STYLES (Code Display) + ========================================================================== */ + +.code-view { + font-family: var(--font-mono); + min-height: 100vh; +} + +/* Code Header (Top Bar for file view) */ +.code-header { + height: var(--topbar-height); + background: var(--header-gradient); + display: flex; + align-items: center; + padding: 0 16px; + gap: 16px; + box-shadow: 0 2px 10px rgba(55, 118, 171, 0.25); + border-bottom: 2px solid var(--python-gold); + position: sticky; + top: 0; + z-index: 100; +} + +.code-header-content { + display: flex; + align-items: center; + justify-content: space-between; + width: 94%; + max-width: 100%; + margin: 0 auto; +} + +.code-header h1 { + font-size: 14px; + font-weight: 600; + color: white; + margin: 0; + font-family: var(--font-mono); + display: flex; + align-items: center; + gap: 8px; +} + +/* File Stats Bar */ +.file-stats { + background: var(--bg-secondary); + padding: 16px 24px; + border-bottom: 1px solid var(--border); +} + +.file-stats .stats-grid { + width: 94%; + max-width: 100%; + margin: 0 auto; + display: grid; + grid-template-columns: repeat(auto-fit, minmax(120px, 1fr)); + gap: 12px; +} + +.stat-item { + background: var(--bg-primary); + padding: 12px; + border-radius: 8px; + box-shadow: var(--shadow-sm); + text-align: center; + border: 1px solid var(--border); + transition: all var(--transition-fast); +} + +.stat-item:hover { + transform: translateY(-2px); + box-shadow: var(--shadow-md); + border-color: var(--accent); +} + +.stat-item .stat-value { + font-size: 1.4em; + font-weight: 700; + color: var(--accent); +} + +.stat-item .stat-label { + color: var(--text-muted); + font-size: 10px; + margin-top: 2px; +} + +/* Legend */ +.legend { + background: var(--bg-secondary); + padding: 12px 24px; + border-bottom: 1px solid var(--border); +} + +.legend-content { + width: 94%; + max-width: 100%; + margin: 0 auto; + display: flex; + align-items: center; + gap: 20px; + flex-wrap: wrap; +} + +.legend-title { + font-weight: 600; + color: var(--text-primary); + font-size: 13px; + font-family: var(--font-sans); +} + +.legend-gradient { + flex: 1; + max-width: 300px; + height: 24px; + background: linear-gradient(90deg, + var(--bg-tertiary) 0%, + var(--heat-2) 25%, + var(--heat-4) 50%, + var(--heat-6) 75%, + var(--heat-8) 100% + ); + border-radius: 4px; + border: 1px solid var(--border); +} + +.legend-labels { + display: flex; + gap: 12px; + font-size: 11px; + color: var(--text-muted); + font-family: var(--font-sans); +} + +/* Toggle Switch Styles */ +.toggle-switch { + display: inline-flex; + align-items: center; + gap: 8px; + cursor: pointer; + user-select: none; + font-family: var(--font-sans); + transition: opacity var(--transition-fast); +} + +.toggle-switch:hover { + opacity: 0.85; +} + +.toggle-switch .toggle-label { + font-size: 11px; + font-weight: 500; + color: var(--text-muted); + min-width: 55px; + text-align: right; + transition: color var(--transition-fast); +} + +.toggle-switch .toggle-label:last-child { + text-align: left; +} + +.toggle-switch .toggle-label.active { + color: var(--text-primary); + font-weight: 600; +} + +.toggle-track { + position: relative; + width: 36px; + height: 20px; + background: var(--bg-tertiary); + border: 2px solid var(--border); + border-radius: 12px; + transition: all var(--transition-fast); + box-shadow: inset var(--shadow-sm); +} + +.toggle-track:hover { + border-color: var(--text-muted); +} + +.toggle-track.on { + background: var(--accent); + border-color: var(--accent); + box-shadow: 0 0 8px var(--accent-glow); +} + +.toggle-track::after { + content: ''; + position: absolute; + top: 1px; + left: 1px; + width: 14px; + height: 14px; + background: white; + border-radius: 50%; + box-shadow: var(--shadow-sm); + transition: all var(--transition-fast); +} + +.toggle-track.on::after { + transform: translateX(16px); + box-shadow: var(--shadow-md); +} + +/* Specific toggle overrides */ +#toggle-color-mode .toggle-track.on { + background: #8e44ad; + border-color: #8e44ad; + box-shadow: 0 0 8px rgba(142, 68, 173, 0.3); +} + +#toggle-cold .toggle-track.on { + background: #e67e22; + border-color: #e67e22; + box-shadow: 0 0 8px rgba(230, 126, 34, 0.3); +} + +/* Code Container */ +.code-container { + width: 94%; + max-width: 100%; + margin: 16px auto; + background: var(--bg-primary); + border: 1px solid var(--border); + border-radius: 8px 8px 8px 8px; + box-shadow: var(--shadow-sm); + /* Allow horizontal scroll for long lines, but don't clip sticky header */ +} + +/* Code Header Row */ +.code-header-row { + position: sticky; + top: var(--topbar-height); + z-index: 50; + display: flex; + background: var(--bg-secondary); + border-bottom: 2px solid var(--border); + font-weight: 700; + font-size: 11px; + color: var(--text-muted); + text-transform: uppercase; + letter-spacing: 0.5px; + border-radius: 8px 8px 0 0; +} + +.header-line-number { + flex-shrink: 0; + width: 60px; + padding: 8px 10px; + text-align: right; + border-right: 1px solid var(--border); +} + +.header-samples-self, +.header-samples-cumulative { + flex-shrink: 0; + width: 90px; + padding: 8px 10px; + text-align: right; + border-right: 1px solid var(--border); +} + +.header-samples-self { + color: var(--heat-8); +} + +.header-samples-cumulative { + color: var(--accent); +} + +.header-content { + flex: 1; + padding: 8px 15px; +} + +/* Code Lines */ +.code-line { + position: relative; + display: flex; + min-height: 20px; + line-height: 20px; + font-size: 13px; + transition: background var(--transition-fast); + scroll-margin-top: calc(var(--topbar-height) + 50px); +} + +.code-line:hover { + filter: brightness(0.97); +} + +[data-theme="dark"] .code-line:hover { + filter: brightness(1.1); +} + +.line-number { + flex-shrink: 0; + width: 60px; + padding: 0 10px; + text-align: right; + color: var(--text-muted); + background: var(--bg-secondary); + border-right: 1px solid var(--border); + user-select: none; + transition: all var(--transition-fast); +} + +.line-number:hover { + background: var(--accent); + color: white; + cursor: pointer; +} + +.line-samples-self, +.line-samples-cumulative { + flex-shrink: 0; + width: 90px; + padding: 0 10px; + text-align: right; + background: var(--bg-secondary); + border-right: 1px solid var(--border); + font-weight: 600; + user-select: none; + font-size: 12px; +} + +.line-samples-self { + color: var(--heat-8); +} + +.line-samples-cumulative { + color: var(--accent); +} + +.line-content { + flex: 1; + padding: 0 15px; + white-space: pre; + overflow-x: auto; +} + +/* Scrollbar Styling */ +.line-content::-webkit-scrollbar { + height: 6px; +} + +.line-content::-webkit-scrollbar-thumb { + background: var(--border); + border-radius: 3px; +} + +.line-content::-webkit-scrollbar-thumb:hover { + background: var(--text-muted); +} + +/* Navigation Buttons */ +.line-nav-buttons { + position: absolute; + right: 8px; + top: 50%; + transform: translateY(-50%); + display: flex; + gap: 4px; + align-items: center; +} + +.nav-btn { + padding: 2px 6px; + font-size: 12px; + font-weight: 500; + border: 1px solid var(--accent); + border-radius: 4px; + background: var(--bg-primary); + color: var(--accent); + cursor: pointer; + transition: all var(--transition-fast); + user-select: none; + line-height: 1; +} + +.nav-btn:hover:not(:disabled) { + background: var(--accent); + color: white; + transform: translateY(-1px); + box-shadow: var(--shadow-sm); +} + +.nav-btn:active:not(:disabled) { + transform: translateY(0); +} + +.nav-btn:disabled { + opacity: 0.3; + cursor: not-allowed; + color: var(--text-muted); + background: var(--bg-secondary); + border-color: var(--border); +} + +.nav-btn.caller { + color: var(--nav-caller); + border-color: var(--nav-caller); +} + +.nav-btn.callee { + color: var(--nav-callee); + border-color: var(--nav-callee); +} + +.nav-btn.caller:hover:not(:disabled) { + background: var(--nav-caller-hover); + color: white; +} + +.nav-btn.callee:hover:not(:disabled) { + background: var(--nav-callee-hover); + color: white; +} + +/* Highlighted target line */ +.code-line:target { + animation: highlight-line 2s ease-out; +} + +@keyframes highlight-line { + 0% { + background: rgba(255, 212, 59, 0.6) !important; + outline: 3px solid var(--python-gold); + outline-offset: -3px; + } + 50% { + background: rgba(255, 212, 59, 0.5) !important; + outline: 3px solid var(--python-gold); + outline-offset: -3px; + } + 100% { + background: inherit; + outline: 3px solid transparent; + outline-offset: -3px; + } +} + +/* Popup menu for multiple callees */ +.callee-menu { + position: absolute; + background: var(--bg-primary); + border: 1px solid var(--border); + border-radius: 8px; + box-shadow: var(--shadow-lg); + padding: 8px; + z-index: 1000; + min-width: 250px; + max-width: 400px; + max-height: 300px; + overflow-y: auto; +} + +.callee-menu-header { + font-weight: 600; + color: var(--text-primary); + margin-bottom: 8px; + padding-bottom: 8px; + border-bottom: 1px solid var(--border); + font-size: 13px; + font-family: var(--font-sans); +} + +.callee-menu-item { + padding: 8px; + margin: 4px 0; + border-radius: 6px; + cursor: pointer; + transition: background var(--transition-fast); + display: flex; + flex-direction: column; + gap: 4px; +} + +.callee-menu-item:hover { + background: var(--bg-secondary); +} + +.callee-menu-func { + font-weight: 500; + color: var(--accent); + font-size: 12px; +} + +.callee-menu-file { + font-size: 11px; + color: var(--text-muted); +} + +.count-badge { + display: inline-block; + background: var(--accent); + color: white; + font-size: 10px; + padding: 2px 6px; + border-radius: 4px; + font-weight: 600; + margin-left: 6px; +} + +/* Callee menu scrollbar */ +.callee-menu::-webkit-scrollbar { + width: 6px; +} + +.callee-menu::-webkit-scrollbar-track { + background: var(--bg-secondary); + border-radius: 3px; +} + +.callee-menu::-webkit-scrollbar-thumb { + background: var(--border); + border-radius: 3px; +} + +/* -------------------------------------------------------------------------- + Scroll Minimap Marker + -------------------------------------------------------------------------- */ + +#scroll_marker { + position: fixed; + z-index: 1000; + right: 0; + top: 0; + width: 12px; + height: 100%; + background: var(--bg-secondary); + border-left: 1px solid var(--border); + pointer-events: none; +} + +#scroll_marker .marker { + position: absolute; + min-height: 3px; + width: 100%; + pointer-events: none; +} + +#scroll_marker .marker.cold { + background: var(--heat-2); +} + +#scroll_marker .marker.warm { + background: var(--heat-5); +} + +#scroll_marker .marker.hot { + background: var(--heat-7); +} + +#scroll_marker .marker.vhot { + background: var(--heat-8); +} + +/* -------------------------------------------------------------------------- + Responsive (Heatmap-specific) + -------------------------------------------------------------------------- */ + +@media (max-width: 1100px) { + .stats-summary { + grid-template-columns: repeat(2, 1fr); + } +} + +@media (max-width: 900px) { + .main-content { + padding: 16px; + } +} + +@media (max-width: 600px) { + .stats-summary { + grid-template-columns: 1fr; + } + + .file-stats .stats-grid { + grid-template-columns: repeat(2, 1fr); + } + + .legend-content { + flex-direction: column; + gap: 12px; + } + + .legend-gradient { + width: 100%; + max-width: none; + } +} diff --git a/Lib/profiling/sampling/_heatmap_assets/heatmap.js b/Lib/profiling/sampling/_heatmap_assets/heatmap.js new file mode 100644 index 00000000000000..ccf823863638dd --- /dev/null +++ b/Lib/profiling/sampling/_heatmap_assets/heatmap.js @@ -0,0 +1,349 @@ +// Tachyon Profiler - Heatmap JavaScript +// Interactive features for the heatmap visualization +// Aligned with Flamegraph viewer design patterns + +// ============================================================================ +// State Management +// ============================================================================ + +let currentMenu = null; +let colorMode = 'self'; // 'self' or 'cumulative' - default to self +let coldCodeHidden = false; + +// ============================================================================ +// Theme Support +// ============================================================================ + +function toggleTheme() { + const html = document.documentElement; + const current = html.getAttribute('data-theme') || 'light'; + const next = current === 'light' ? 'dark' : 'light'; + html.setAttribute('data-theme', next); + localStorage.setItem('heatmap-theme', next); + + // Update theme button icon + const btn = document.getElementById('theme-btn'); + if (btn) { + btn.innerHTML = next === 'dark' ? '☼' : '☾'; // sun or moon + } + + // Rebuild scroll marker with new theme colors + buildScrollMarker(); +} + +function restoreUIState() { + // Restore theme + const savedTheme = localStorage.getItem('heatmap-theme'); + if (savedTheme) { + document.documentElement.setAttribute('data-theme', savedTheme); + const btn = document.getElementById('theme-btn'); + if (btn) { + btn.innerHTML = savedTheme === 'dark' ? '☼' : '☾'; + } + } +} + +// ============================================================================ +// Utility Functions +// ============================================================================ + +function createElement(tag, className, textContent = '') { + const el = document.createElement(tag); + if (className) el.className = className; + if (textContent) el.textContent = textContent; + return el; +} + +function calculateMenuPosition(buttonRect, menuWidth, menuHeight) { + const viewport = { width: window.innerWidth, height: window.innerHeight }; + const scroll = { + x: window.pageXOffset || document.documentElement.scrollLeft, + y: window.pageYOffset || document.documentElement.scrollTop + }; + + const left = buttonRect.right + menuWidth + 10 < viewport.width + ? buttonRect.right + scroll.x + 10 + : Math.max(scroll.x + 10, buttonRect.left + scroll.x - menuWidth - 10); + + const top = buttonRect.bottom + menuHeight + 10 < viewport.height + ? buttonRect.bottom + scroll.y + 5 + : Math.max(scroll.y + 10, buttonRect.top + scroll.y - menuHeight - 10); + + return { left, top }; +} + +// ============================================================================ +// Menu Management +// ============================================================================ + +function closeMenu() { + if (currentMenu) { + currentMenu.remove(); + currentMenu = null; + } +} + +function showNavigationMenu(button, items, title) { + closeMenu(); + + const menu = createElement('div', 'callee-menu'); + menu.appendChild(createElement('div', 'callee-menu-header', title)); + + items.forEach(linkData => { + const item = createElement('div', 'callee-menu-item'); + + const funcDiv = createElement('div', 'callee-menu-func'); + funcDiv.textContent = linkData.func; + + if (linkData.count !== undefined && linkData.count > 0) { + const countBadge = createElement('span', 'count-badge'); + countBadge.textContent = linkData.count.toLocaleString(); + countBadge.title = `${linkData.count.toLocaleString()} samples`; + funcDiv.appendChild(document.createTextNode(' ')); + funcDiv.appendChild(countBadge); + } + + item.appendChild(funcDiv); + item.appendChild(createElement('div', 'callee-menu-file', linkData.file)); + item.addEventListener('click', () => window.location.href = linkData.link); + menu.appendChild(item); + }); + + const pos = calculateMenuPosition(button.getBoundingClientRect(), 350, 300); + menu.style.left = `${pos.left}px`; + menu.style.top = `${pos.top}px`; + + document.body.appendChild(menu); + currentMenu = menu; +} + +// ============================================================================ +// Navigation +// ============================================================================ + +function handleNavigationClick(button, e) { + e.stopPropagation(); + + const navData = button.getAttribute('data-nav'); + if (navData) { + window.location.href = JSON.parse(navData).link; + return; + } + + const navMulti = button.getAttribute('data-nav-multi'); + if (navMulti) { + const items = JSON.parse(navMulti); + const title = button.classList.contains('caller') ? 'Choose a caller:' : 'Choose a callee:'; + showNavigationMenu(button, items, title); + } +} + +function scrollToTargetLine() { + if (!window.location.hash) return; + const target = document.querySelector(window.location.hash); + if (target) { + target.scrollIntoView({ behavior: 'smooth', block: 'start' }); + } +} + +// ============================================================================ +// Sample Count & Intensity +// ============================================================================ + +function getSampleCount(line) { + let text; + if (colorMode === 'self') { + text = line.querySelector('.line-samples-self')?.textContent.trim().replace(/,/g, ''); + } else { + text = line.querySelector('.line-samples-cumulative')?.textContent.trim().replace(/,/g, ''); + } + return parseInt(text) || 0; +} + +function getIntensityClass(ratio) { + if (ratio > 0.75) return 'vhot'; + if (ratio > 0.5) return 'hot'; + if (ratio > 0.25) return 'warm'; + return 'cold'; +} + +// ============================================================================ +// Scroll Minimap +// ============================================================================ + +function buildScrollMarker() { + const existing = document.getElementById('scroll_marker'); + if (existing) existing.remove(); + + if (document.body.scrollHeight <= window.innerHeight) return; + + const allLines = document.querySelectorAll('.code-line'); + const lines = Array.from(allLines).filter(line => line.style.display !== 'none'); + const markerScale = window.innerHeight / document.body.scrollHeight; + const lineHeight = Math.min(Math.max(3, window.innerHeight / lines.length), 10); + const maxSamples = Math.max(...Array.from(lines, getSampleCount)); + + const scrollMarker = createElement('div', ''); + scrollMarker.id = 'scroll_marker'; + + let prevLine = -99, lastMark, lastTop; + + lines.forEach((line, index) => { + const samples = getSampleCount(line); + if (samples === 0) return; + + const lineTop = Math.floor(line.offsetTop * markerScale); + const lineNumber = index + 1; + const intensityClass = maxSamples > 0 ? getIntensityClass(samples / maxSamples) : 'cold'; + + if (lineNumber === prevLine + 1 && lastMark?.classList.contains(intensityClass)) { + lastMark.style.height = `${lineTop + lineHeight - lastTop}px`; + } else { + lastMark = createElement('div', `marker ${intensityClass}`); + lastMark.style.height = `${lineHeight}px`; + lastMark.style.top = `${lineTop}px`; + scrollMarker.appendChild(lastMark); + lastTop = lineTop; + } + + prevLine = lineNumber; + }); + + document.body.appendChild(scrollMarker); +} + +// ============================================================================ +// Toggle Controls +// ============================================================================ + +function updateToggleUI(toggleId, isOn) { + const toggle = document.getElementById(toggleId); + if (toggle) { + const track = toggle.querySelector('.toggle-track'); + const labels = toggle.querySelectorAll('.toggle-label'); + if (isOn) { + track.classList.add('on'); + labels[0].classList.remove('active'); + labels[1].classList.add('active'); + } else { + track.classList.remove('on'); + labels[0].classList.add('active'); + labels[1].classList.remove('active'); + } + } +} + +function toggleColdCode() { + coldCodeHidden = !coldCodeHidden; + applyHotFilter(); + updateToggleUI('toggle-cold', coldCodeHidden); + buildScrollMarker(); +} + +function applyHotFilter() { + const lines = document.querySelectorAll('.code-line'); + + lines.forEach(line => { + const selfSamples = line.querySelector('.line-samples-self')?.textContent.trim(); + const cumulativeSamples = line.querySelector('.line-samples-cumulative')?.textContent.trim(); + + let isCold; + if (colorMode === 'self') { + isCold = !selfSamples || selfSamples === ''; + } else { + isCold = !cumulativeSamples || cumulativeSamples === ''; + } + + if (isCold) { + line.style.display = coldCodeHidden ? 'none' : 'flex'; + } else { + line.style.display = 'flex'; + } + }); +} + +function toggleColorMode() { + colorMode = colorMode === 'self' ? 'cumulative' : 'self'; + const lines = document.querySelectorAll('.code-line'); + + lines.forEach(line => { + let bgColor; + if (colorMode === 'self') { + bgColor = line.getAttribute('data-self-color'); + } else { + bgColor = line.getAttribute('data-cumulative-color'); + } + + if (bgColor) { + line.style.background = bgColor; + } + }); + + updateToggleUI('toggle-color-mode', colorMode === 'cumulative'); + + if (coldCodeHidden) { + applyHotFilter(); + } + + buildScrollMarker(); +} + +// ============================================================================ +// Initialization +// ============================================================================ + +document.addEventListener('DOMContentLoaded', function() { + // Restore UI state (theme, etc.) + restoreUIState(); + + // Apply background colors + document.querySelectorAll('.code-line[data-bg-color]').forEach(line => { + const bgColor = line.getAttribute('data-bg-color'); + if (bgColor) { + line.style.background = bgColor; + } + }); + + // Initialize navigation buttons + document.querySelectorAll('.nav-btn').forEach(button => { + button.addEventListener('click', e => handleNavigationClick(button, e)); + }); + + // Initialize line number permalink handlers + document.querySelectorAll('.line-number').forEach(lineNum => { + lineNum.style.cursor = 'pointer'; + lineNum.addEventListener('click', e => { + window.location.hash = `line-${e.target.textContent.trim()}`; + }); + }); + + // Initialize toggle buttons + const toggleColdBtn = document.getElementById('toggle-cold'); + if (toggleColdBtn) { + toggleColdBtn.addEventListener('click', toggleColdCode); + } + + const colorModeBtn = document.getElementById('toggle-color-mode'); + if (colorModeBtn) { + colorModeBtn.addEventListener('click', toggleColorMode); + } + + // Build scroll marker + setTimeout(buildScrollMarker, 200); + + // Setup scroll-to-line behavior + setTimeout(scrollToTargetLine, 100); +}); + +// Close menu when clicking outside +document.addEventListener('click', e => { + if (currentMenu && !currentMenu.contains(e.target) && !e.target.classList.contains('nav-btn')) { + closeMenu(); + } +}); + +// Handle hash changes +window.addEventListener('hashchange', () => setTimeout(scrollToTargetLine, 50)); + +// Rebuild scroll marker on resize +window.addEventListener('resize', buildScrollMarker); diff --git a/Lib/profiling/sampling/_heatmap_assets/heatmap_index.js b/Lib/profiling/sampling/_heatmap_assets/heatmap_index.js new file mode 100644 index 00000000000000..5f3e65c3310884 --- /dev/null +++ b/Lib/profiling/sampling/_heatmap_assets/heatmap_index.js @@ -0,0 +1,111 @@ +// Tachyon Profiler - Heatmap Index JavaScript +// Index page specific functionality + +// ============================================================================ +// Theme Support +// ============================================================================ + +function toggleTheme() { + const html = document.documentElement; + const current = html.getAttribute('data-theme') || 'light'; + const next = current === 'light' ? 'dark' : 'light'; + html.setAttribute('data-theme', next); + localStorage.setItem('heatmap-theme', next); + + // Update theme button icon + const btn = document.getElementById('theme-btn'); + if (btn) { + btn.innerHTML = next === 'dark' ? '☼' : '☾'; // sun or moon + } +} + +function restoreUIState() { + // Restore theme + const savedTheme = localStorage.getItem('heatmap-theme'); + if (savedTheme) { + document.documentElement.setAttribute('data-theme', savedTheme); + const btn = document.getElementById('theme-btn'); + if (btn) { + btn.innerHTML = savedTheme === 'dark' ? '☼' : '☾'; + } + } +} + +// ============================================================================ +// Type Section Toggle (stdlib, project, etc) +// ============================================================================ + +function toggleTypeSection(header) { + const section = header.parentElement; + const content = section.querySelector('.type-content'); + const icon = header.querySelector('.type-icon'); + + if (content.style.display === 'none') { + content.style.display = 'block'; + icon.textContent = '\u25BC'; + } else { + content.style.display = 'none'; + icon.textContent = '\u25B6'; + } +} + +// ============================================================================ +// Folder Toggle +// ============================================================================ + +function toggleFolder(header) { + const folder = header.parentElement; + const content = folder.querySelector('.folder-content'); + const icon = header.querySelector('.folder-icon'); + + if (content.style.display === 'none') { + content.style.display = 'block'; + icon.textContent = '\u25BC'; + folder.classList.remove('collapsed'); + } else { + content.style.display = 'none'; + icon.textContent = '\u25B6'; + folder.classList.add('collapsed'); + } +} + +// ============================================================================ +// Expand/Collapse All +// ============================================================================ + +function expandAll() { + // Expand all type sections + document.querySelectorAll('.type-section').forEach(section => { + const content = section.querySelector('.type-content'); + const icon = section.querySelector('.type-icon'); + content.style.display = 'block'; + icon.textContent = '\u25BC'; + }); + + // Expand all folders + document.querySelectorAll('.folder-node').forEach(folder => { + const content = folder.querySelector('.folder-content'); + const icon = folder.querySelector('.folder-icon'); + content.style.display = 'block'; + icon.textContent = '\u25BC'; + folder.classList.remove('collapsed'); + }); +} + +function collapseAll() { + document.querySelectorAll('.folder-node').forEach(folder => { + const content = folder.querySelector('.folder-content'); + const icon = folder.querySelector('.folder-icon'); + content.style.display = 'none'; + icon.textContent = '\u25B6'; + folder.classList.add('collapsed'); + }); +} + +// ============================================================================ +// Initialization +// ============================================================================ + +document.addEventListener('DOMContentLoaded', function() { + restoreUIState(); +}); diff --git a/Lib/profiling/sampling/_heatmap_assets/heatmap_index_template.html b/Lib/profiling/sampling/_heatmap_assets/heatmap_index_template.html new file mode 100644 index 00000000000000..b71bd94c66166a --- /dev/null +++ b/Lib/profiling/sampling/_heatmap_assets/heatmap_index_template.html @@ -0,0 +1,118 @@ + + + + + + Tachyon Profiler - Heatmap Report + + + +
+ +
+
+ Tachyon + + Heatmap Report +
+
+ +
+
+ + +
+ +
+
+
📄
+
+ + Files Profiled +
+
+
+
+
📊
+
+ + Total Snapshots +
+
+
+
+
+
+ + Duration +
+
+
+
+
+
+ + Samples/sec +
+
+
+
+
+
+
+ Error Rate +
+ +
+
+
+
+
+
+
+
+
💥
+ Missed Samples +
+ +
+
+
+
+
+
+ + +
+

Profiled Files

+
+ +
+ + +
+ +
+ +
+
+ + +
+ + Tachyon Profiler + + + Python Sampling Profiler + +
+
+ + + + diff --git a/Lib/profiling/sampling/_heatmap_assets/heatmap_pyfile_template.html b/Lib/profiling/sampling/_heatmap_assets/heatmap_pyfile_template.html new file mode 100644 index 00000000000000..d8b26adfb0243f --- /dev/null +++ b/Lib/profiling/sampling/_heatmap_assets/heatmap_pyfile_template.html @@ -0,0 +1,96 @@ + + + + + + <!-- FILENAME --> - Heatmap + + + +
+ +
+
+ Tachyon + + +
+
+ Back to Index + +
+
+ + +
+
+
+
+
Self Samples
+
+
+
+
Cumulative
+
+
+
+
Lines Hit
+
+
+
%
+
% of Total
+
+
+
+
Max Self
+
+
+
+
Max Total
+
+
+
+ + +
+
+ Intensity: +
+
+ Cold + + Hot +
+
+ Self Time +
+ Total Time +
+
+ Show All +
+ Hot Only +
+
+
+ + +
+
+
Line
+
Self
+
Total
+
Code
+
+ +
+
+ + + + diff --git a/Lib/profiling/sampling/_shared_assets/base.css b/Lib/profiling/sampling/_shared_assets/base.css new file mode 100644 index 00000000000000..20516913496cbe --- /dev/null +++ b/Lib/profiling/sampling/_shared_assets/base.css @@ -0,0 +1,369 @@ +/* ========================================================================== + Python Profiler - Shared CSS Foundation + Design system shared between Flamegraph and Heatmap viewers + ========================================================================== */ + +/* -------------------------------------------------------------------------- + CSS Variables & Theme System + -------------------------------------------------------------------------- */ + +:root { + /* Typography */ + --font-sans: "Source Sans Pro", "Lucida Grande", "Lucida Sans Unicode", + "Geneva", "Verdana", sans-serif; + --font-mono: 'SF Mono', 'Monaco', 'Consolas', 'Liberation Mono', monospace; + + /* Python brand colors (theme-independent) */ + --python-blue: #3776ab; + --python-blue-light: #4584bb; + --python-blue-lighter: #5592cc; + --python-gold: #ffd43b; + --python-gold-dark: #ffcd02; + --python-gold-light: #ffdc5c; + + /* Heat palette - defined per theme below */ + + /* Layout */ + --sidebar-width: 280px; + --sidebar-collapsed: 44px; + --topbar-height: 56px; + --statusbar-height: 32px; + + /* Transitions */ + --transition-fast: 0.15s ease; + --transition-normal: 0.25s ease; +} + +/* Light theme (default) */ +:root, [data-theme="light"] { + --bg-primary: #ffffff; + --bg-secondary: #f8f9fa; + --bg-tertiary: #e9ecef; + --border: #e9ecef; + --border-subtle: #f0f2f5; + + --text-primary: #2e3338; + --text-secondary: #5a6c7d; + --text-muted: #8b949e; + + --accent: #3776ab; + --accent-hover: #2d5aa0; + --accent-glow: rgba(55, 118, 171, 0.15); + + --shadow-sm: 0 1px 3px rgba(0, 0, 0, 0.08); + --shadow-md: 0 4px 12px rgba(0, 0, 0, 0.1); + --shadow-lg: 0 8px 24px rgba(0, 0, 0, 0.15); + + --header-gradient: linear-gradient(135deg, #3776ab 0%, #4584bb 100%); + + /* Light mode heat palette - blue to yellow to orange to red (cold to hot) */ + --heat-1: #d6e9f8; + --heat-2: #a8d0ef; + --heat-3: #7ba3d1; + --heat-4: #ffe6a8; + --heat-5: #ffd43b; + --heat-6: #ffb84d; + --heat-7: #ff9966; + --heat-8: #ff6347; + + /* Code view specific */ + --code-bg: #ffffff; + --code-bg-line: #f8f9fa; + --code-border: #e9ecef; + --code-text: #2e3338; + --code-text-muted: #8b949e; + --code-accent: #3776ab; + + /* Navigation colors */ + --nav-caller: #2563eb; + --nav-caller-hover: #1d4ed8; + --nav-callee: #dc2626; + --nav-callee-hover: #b91c1c; +} + +/* Dark theme */ +[data-theme="dark"] { + --bg-primary: #0d1117; + --bg-secondary: #161b22; + --bg-tertiary: #21262d; + --border: #30363d; + --border-subtle: #21262d; + + --text-primary: #e6edf3; + --text-secondary: #8b949e; + --text-muted: #6e7681; + + --accent: #58a6ff; + --accent-hover: #79b8ff; + --accent-glow: rgba(88, 166, 255, 0.15); + + --shadow-sm: 0 1px 3px rgba(0, 0, 0, 0.3); + --shadow-md: 0 4px 12px rgba(0, 0, 0, 0.4); + --shadow-lg: 0 8px 24px rgba(0, 0, 0, 0.5); + + --header-gradient: linear-gradient(135deg, #21262d 0%, #30363d 100%); + + /* Dark mode heat palette - dark blue to teal to yellow to orange (cold to hot) */ + --heat-1: #1e3a5f; + --heat-2: #2d5580; + --heat-3: #4a7ba7; + --heat-4: #5a9fa8; + --heat-5: #7ec488; + --heat-6: #c4de6a; + --heat-7: #f4d44d; + --heat-8: #ff6b35; + + /* Code view specific - dark mode */ + --code-bg: #0d1117; + --code-bg-line: #161b22; + --code-border: #30363d; + --code-text: #e6edf3; + --code-text-muted: #6e7681; + --code-accent: #58a6ff; + + /* Navigation colors - dark theme friendly */ + --nav-caller: #58a6ff; + --nav-caller-hover: #4184e4; + --nav-callee: #f87171; + --nav-callee-hover: #e53e3e; +} + +/* -------------------------------------------------------------------------- + Base Styles + -------------------------------------------------------------------------- */ + +*, *::before, *::after { + box-sizing: border-box; +} + +html, body { + margin: 0; + padding: 0; +} + +body { + font-family: var(--font-sans); + font-size: 14px; + line-height: 1.6; + color: var(--text-primary); + background: var(--bg-primary); + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; + transition: background var(--transition-normal), color var(--transition-normal); +} + +/* -------------------------------------------------------------------------- + Layout Structure + -------------------------------------------------------------------------- */ + +.app-layout { + display: flex; + flex-direction: column; +} + +/* -------------------------------------------------------------------------- + Top Bar + -------------------------------------------------------------------------- */ + +.top-bar { + height: var(--topbar-height); + background: var(--header-gradient); + display: flex; + align-items: center; + padding: 0 16px; + gap: 16px; + flex-shrink: 0; + box-shadow: 0 2px 10px rgba(55, 118, 171, 0.25); + border-bottom: 2px solid var(--python-gold); +} + +/* Brand / Logo */ +.brand { + display: flex; + align-items: center; + gap: 12px; + color: white; + text-decoration: none; + flex-shrink: 0; +} + +.brand-logo { + display: flex; + align-items: center; + justify-content: center; + width: 28px; + height: 28px; + flex-shrink: 0; +} + +/* Style the inlined SVG/img inside brand-logo */ +.brand-logo svg, +.brand-logo img { + width: 28px; + height: 28px; + display: block; + object-fit: contain; + filter: drop-shadow(0 1px 2px rgba(0, 0, 0, 0.2)); +} + +.brand-info { + display: flex; + flex-direction: column; + line-height: 1.15; +} + +.brand-text { + font-weight: 700; + font-size: 16px; + letter-spacing: -0.3px; + text-shadow: 0 1px 2px rgba(0, 0, 0, 0.15); +} + +.brand-subtitle { + font-weight: 500; + font-size: 10px; + opacity: 0.9; + text-transform: uppercase; + letter-spacing: 0.5px; +} + +.brand-divider { + width: 1px; + height: 16px; + background: rgba(255, 255, 255, 0.3); +} + +/* Toolbar */ +.toolbar { + display: flex; + align-items: center; + gap: 6px; + margin-left: auto; +} + +.toolbar-btn { + display: flex; + align-items: center; + justify-content: center; + width: 32px; + height: 32px; + padding: 0; + font-size: 15px; + color: white; + background: rgba(255, 255, 255, 0.12); + border: 1px solid rgba(255, 255, 255, 0.18); + border-radius: 6px; + cursor: pointer; + transition: all var(--transition-fast); +} + +.toolbar-btn:hover { + background: rgba(255, 255, 255, 0.22); + border-color: rgba(255, 255, 255, 0.35); +} + +.toolbar-btn:active { + transform: scale(0.95); +} + +/* -------------------------------------------------------------------------- + Status Bar + -------------------------------------------------------------------------- */ + +.status-bar { + height: var(--statusbar-height); + background: var(--bg-secondary); + border-top: 1px solid var(--border); + display: flex; + align-items: center; + padding: 0 16px; + gap: 16px; + font-family: var(--font-mono); + font-size: 11px; + color: var(--text-secondary); + flex-shrink: 0; +} + +.status-item { + display: flex; + align-items: center; + gap: 5px; +} + +.status-item::before { + content: ''; + width: 4px; + height: 4px; + background: var(--python-gold); + border-radius: 50%; +} + +.status-item:first-child::before { + display: none; +} + +.status-label { + color: var(--text-muted); +} + +.status-value { + color: var(--text-primary); + font-weight: 500; +} + +.status-value.accent { + color: var(--accent); + font-weight: 600; +} + +/* -------------------------------------------------------------------------- + Animations + -------------------------------------------------------------------------- */ + +@keyframes fadeIn { + from { opacity: 0; } + to { opacity: 1; } +} + +@keyframes slideUp { + from { + opacity: 0; + transform: translateY(12px); + } + to { + opacity: 1; + transform: translateY(0); + } +} + +@keyframes shimmer { + 0% { left: -100%; } + 100% { left: 100%; } +} + +/* -------------------------------------------------------------------------- + Focus States (Accessibility) + -------------------------------------------------------------------------- */ + +button:focus-visible, +select:focus-visible, +input:focus-visible { + outline: 2px solid var(--python-gold); + outline-offset: 2px; +} + +/* -------------------------------------------------------------------------- + Shared Responsive + -------------------------------------------------------------------------- */ + +@media (max-width: 900px) { + .brand-subtitle { + display: none; + } +} + +@media (max-width: 600px) { + .toolbar-btn:not(.theme-toggle) { + display: none; + } +} diff --git a/Lib/profiling/sampling/cli.py b/Lib/profiling/sampling/cli.py index aede6a4d3e9f1b..5c0e39d77371ef 100644 --- a/Lib/profiling/sampling/cli.py +++ b/Lib/profiling/sampling/cli.py @@ -9,6 +9,7 @@ from .sample import sample, sample_live from .pstats_collector import PstatsCollector from .stack_collector import CollapsedStackCollector, FlamegraphCollector +from .heatmap_collector import HeatmapCollector from .gecko_collector import GeckoCollector from .constants import ( PROFILING_MODE_ALL, @@ -71,6 +72,7 @@ class CustomFormatter( "collapsed": "txt", "flamegraph": "html", "gecko": "json", + "heatmap": "html", } COLLECTOR_MAP = { @@ -78,6 +80,7 @@ class CustomFormatter( "collapsed": CollapsedStackCollector, "flamegraph": FlamegraphCollector, "gecko": GeckoCollector, + "heatmap": HeatmapCollector, } @@ -238,14 +241,21 @@ def _add_format_options(parser): dest="format", help="Generate Gecko format for Firefox Profiler", ) + format_group.add_argument( + "--heatmap", + action="store_const", + const="heatmap", + dest="format", + help="Generate interactive HTML heatmap visualization with line-level sample counts", + ) parser.set_defaults(format="pstats") output_group.add_argument( "-o", "--output", dest="outfile", - help="Save output to a file (default: stdout for pstats, " - "auto-generated filename for other formats)", + help="Output path (default: stdout for pstats, auto-generated for others). " + "For heatmap: directory name (default: heatmap_PID)", ) @@ -327,6 +337,9 @@ def _generate_output_filename(format_type, pid): Generated filename """ extension = FORMAT_EXTENSIONS.get(format_type, "txt") + # For heatmap, use cleaner directory name without extension + if format_type == "heatmap": + return f"heatmap_{pid}" return f"{format_type}.{pid}.{extension}" diff --git a/Lib/profiling/sampling/heatmap_collector.py b/Lib/profiling/sampling/heatmap_collector.py new file mode 100644 index 00000000000000..eb51ce33b28a52 --- /dev/null +++ b/Lib/profiling/sampling/heatmap_collector.py @@ -0,0 +1,1039 @@ +"""Heatmap collector for Python profiling with line-level execution heat visualization.""" + +import base64 +import collections +import html +import importlib.resources +import json +import os +import platform +import site +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Dict, List, Tuple, Optional, Any + +from ._css_utils import get_combined_css +from .stack_collector import StackTraceCollector + + +# ============================================================================ +# Data Classes +# ============================================================================ + +@dataclass +class FileStats: + """Statistics for a single profiled file.""" + filename: str + module_name: str + module_type: str + total_samples: int + total_self_samples: int + num_lines: int + max_samples: int + max_self_samples: int + percentage: float = 0.0 + + +@dataclass +class TreeNode: + """Node in the hierarchical file tree structure.""" + files: List[FileStats] = field(default_factory=list) + samples: int = 0 + count: int = 0 + children: Dict[str, 'TreeNode'] = field(default_factory=dict) + + +@dataclass +class ColorGradient: + """Configuration for heatmap color gradient calculations.""" + # Color stops thresholds + stop_1: float = 0.2 # Blue to cyan transition + stop_2: float = 0.4 # Cyan to green transition + stop_3: float = 0.6 # Green to yellow transition + stop_4: float = 0.8 # Yellow to orange transition + stop_5: float = 1.0 # Orange to red transition + + # Alpha (opacity) values + alpha_very_cold: float = 0.3 + alpha_cold: float = 0.4 + alpha_medium: float = 0.5 + alpha_warm: float = 0.6 + alpha_hot_base: float = 0.7 + alpha_hot_range: float = 0.15 + + # Gradient multiplier + multiplier: int = 5 + + # Cache for calculated colors + cache: Dict[float, Tuple[int, int, int, float]] = field(default_factory=dict) + + +# ============================================================================ +# Module Path Analysis +# ============================================================================ + +def get_python_path_info(): + """Get information about Python installation paths for module extraction. + + Returns: + dict: Dictionary containing stdlib path, site-packages paths, and sys.path entries. + """ + info = { + 'stdlib': None, + 'site_packages': [], + 'sys_path': [] + } + + # Get standard library path from os module location + try: + if hasattr(os, '__file__') and os.__file__: + info['stdlib'] = Path(os.__file__).parent + except (AttributeError, OSError): + pass # Silently continue if we can't determine stdlib path + + # Get site-packages directories + site_packages = [] + try: + site_packages.extend(Path(p) for p in site.getsitepackages()) + except (AttributeError, OSError): + pass # Continue without site packages if unavailable + + # Get user site-packages + try: + user_site = site.getusersitepackages() + if user_site and Path(user_site).exists(): + site_packages.append(Path(user_site)) + except (AttributeError, OSError): + pass # Continue without user site packages + + info['site_packages'] = site_packages + info['sys_path'] = [Path(p) for p in sys.path if p] + + return info + + +def extract_module_name(filename, path_info): + """Extract Python module name and type from file path. + + Args: + filename: Path to the Python file + path_info: Dictionary from get_python_path_info() + + Returns: + tuple: (module_name, module_type) where module_type is one of: + 'stdlib', 'site-packages', 'project', or 'other' + """ + if not filename: + return ('unknown', 'other') + + try: + file_path = Path(filename) + except (ValueError, OSError): + return (str(filename), 'other') + + # Check if it's in stdlib + if path_info['stdlib'] and _is_subpath(file_path, path_info['stdlib']): + try: + rel_path = file_path.relative_to(path_info['stdlib']) + return (_path_to_module(rel_path), 'stdlib') + except ValueError: + pass + + # Check site-packages + for site_pkg in path_info['site_packages']: + if _is_subpath(file_path, site_pkg): + try: + rel_path = file_path.relative_to(site_pkg) + return (_path_to_module(rel_path), 'site-packages') + except ValueError: + continue + + # Check other sys.path entries (project files) + if not str(file_path).startswith(('<', '[')): # Skip special files + for path_entry in path_info['sys_path']: + if _is_subpath(file_path, path_entry): + try: + rel_path = file_path.relative_to(path_entry) + return (_path_to_module(rel_path), 'project') + except ValueError: + continue + + # Fallback: just use the filename + return (_path_to_module(file_path), 'other') + + +def _is_subpath(file_path, parent_path): + try: + file_path.relative_to(parent_path) + return True + except (ValueError, OSError): + return False + + +def _path_to_module(path): + if isinstance(path, str): + path = Path(path) + + # Remove .py extension + if path.suffix == '.py': + path = path.with_suffix('') + + # Convert path separators to dots + parts = path.parts + + # Handle __init__ files - they represent the package itself + if parts and parts[-1] == '__init__': + parts = parts[:-1] + + return '.'.join(parts) if parts else path.stem + + +# ============================================================================ +# Helper Classes +# ============================================================================ + +class _TemplateLoader: + """Loads and caches HTML/CSS/JS templates for heatmap generation.""" + + def __init__(self): + """Load all templates and assets once.""" + self.index_template = None + self.file_template = None + self.index_css = None + self.index_js = None + self.file_css = None + self.file_js = None + self.logo_html = None + + self._load_templates() + + def _load_templates(self): + """Load all template files from _heatmap_assets.""" + try: + template_dir = importlib.resources.files(__package__) + assets_dir = template_dir / "_heatmap_assets" + + # Load HTML templates + self.index_template = (assets_dir / "heatmap_index_template.html").read_text(encoding="utf-8") + self.file_template = (assets_dir / "heatmap_pyfile_template.html").read_text(encoding="utf-8") + + # Load CSS (same file used for both index and file pages) + css_content = get_combined_css("heatmap") + self.index_css = css_content + self.file_css = css_content + + # Load JS + self.index_js = (assets_dir / "heatmap_index.js").read_text(encoding="utf-8") + self.file_js = (assets_dir / "heatmap.js").read_text(encoding="utf-8") + + # Load Python logo + logo_dir = template_dir / "_assets" + try: + png_path = logo_dir / "python-logo-only.png" + b64_logo = base64.b64encode(png_path.read_bytes()).decode("ascii") + self.logo_html = f'' + except (FileNotFoundError, IOError) as e: + self.logo_html = '
' + print(f"Warning: Could not load Python logo: {e}") + + except (FileNotFoundError, IOError) as e: + raise RuntimeError(f"Failed to load heatmap template files: {e}") from e + + +class _TreeBuilder: + """Builds hierarchical tree structure from file statistics.""" + + @staticmethod + def build_file_tree(file_stats: List[FileStats]) -> Dict[str, TreeNode]: + """Build hierarchical tree grouped by module type, then by module structure. + + Args: + file_stats: List of FileStats objects + + Returns: + Dictionary mapping module types to their tree roots + """ + # Group by module type first + type_groups = {'stdlib': [], 'site-packages': [], 'project': [], 'other': []} + for stat in file_stats: + type_groups[stat.module_type].append(stat) + + # Build tree for each type + trees = {} + for module_type, stats in type_groups.items(): + if not stats: + continue + + root_node = TreeNode() + + for stat in stats: + module_name = stat.module_name + parts = module_name.split('.') + + # Navigate/create tree structure + current_node = root_node + for i, part in enumerate(parts): + if i == len(parts) - 1: + # Last part - store the file + current_node.files.append(stat) + else: + # Intermediate part - create or navigate + if part not in current_node.children: + current_node.children[part] = TreeNode() + current_node = current_node.children[part] + + # Calculate aggregate stats for this type's tree + _TreeBuilder._calculate_node_stats(root_node) + trees[module_type] = root_node + + return trees + + @staticmethod + def _calculate_node_stats(node: TreeNode) -> Tuple[int, int]: + """Recursively calculate aggregate statistics for tree nodes. + + Args: + node: TreeNode to calculate stats for + + Returns: + Tuple of (total_samples, file_count) + """ + total_samples = 0 + file_count = 0 + + # Count files at this level + for file_stat in node.files: + total_samples += file_stat.total_samples + file_count += 1 + + # Recursively process children + for child in node.children.values(): + child_samples, child_count = _TreeBuilder._calculate_node_stats(child) + total_samples += child_samples + file_count += child_count + + node.samples = total_samples + node.count = file_count + return total_samples, file_count + + +class _HtmlRenderer: + """Renders hierarchical tree structures as HTML.""" + + def __init__(self, file_index: Dict[str, str], color_gradient: ColorGradient, + calculate_intensity_color_func): + """Initialize renderer with file index and color calculation function. + + Args: + file_index: Mapping from filenames to HTML file names + color_gradient: ColorGradient configuration + calculate_intensity_color_func: Function to calculate colors + """ + self.file_index = file_index + self.color_gradient = color_gradient + self.calculate_intensity_color = calculate_intensity_color_func + self.heatmap_bar_height = 16 + + def render_hierarchical_html(self, trees: Dict[str, TreeNode]) -> str: + """Build hierarchical HTML with type sections and collapsible module folders. + + Args: + trees: Dictionary mapping module types to tree roots + + Returns: + Complete HTML string for all sections + """ + type_names = { + 'stdlib': '📚 Standard Library', + 'site-packages': '📦 Site Packages', + 'project': '🏗️ Project Files', + 'other': '📄 Other Files' + } + + sections = [] + for module_type in ['project', 'stdlib', 'site-packages', 'other']: + if module_type not in trees: + continue + + tree = trees[module_type] + + # Project starts expanded, others start collapsed + is_collapsed = module_type in {'stdlib', 'site-packages', 'other'} + icon = '▶' if is_collapsed else '▼' + content_style = ' style="display: none;"' if is_collapsed else '' + + section_html = f''' +
+
+ {icon} + {type_names[module_type]} + ({tree.count} files, {tree.samples:,} samples) +
+
+''' + + # Render root folders + root_folders = sorted(tree.children.items(), + key=lambda x: x[1].samples, reverse=True) + + for folder_name, folder_node in root_folders: + section_html += self._render_folder(folder_node, folder_name, level=1) + + # Render root files (files not in any module) + if tree.files: + sorted_files = sorted(tree.files, key=lambda x: x.total_samples, reverse=True) + section_html += '
\n' + for stat in sorted_files: + section_html += self._render_file_item(stat, indent=' ') + section_html += '
\n' + + section_html += '
\n
\n' + sections.append(section_html) + + return '\n'.join(sections) + + def _render_folder(self, node: TreeNode, name: str, level: int = 1) -> str: + """Render a single folder node recursively. + + Args: + node: TreeNode to render + name: Display name for the folder + level: Nesting level for indentation + + Returns: + HTML string for this folder and its contents + """ + indent = ' ' * level + parts = [] + + # Render folder header (collapsed by default) + parts.append(f'{indent}') + + return '\n'.join(parts) + + def _render_file_item(self, stat: FileStats, indent: str = '') -> str: + """Render a single file item with heatmap bar. + + Args: + stat: FileStats object + indent: Indentation string + + Returns: + HTML string for file item + """ + full_path = html.escape(stat.filename) + module_name = html.escape(stat.module_name) + + intensity = stat.percentage / 100.0 + r, g, b, alpha = self.calculate_intensity_color(intensity) + bg_color = f"rgba({r}, {g}, {b}, {alpha})" + bar_width = min(stat.percentage, 100) + + html_file = self.file_index[stat.filename] + + return (f'{indent}
\n' + f'{indent} 📄 {module_name}\n' + f'{indent} {stat.total_samples:,} samples\n' + f'{indent}
\n' + f'{indent}
\n') + + +# ============================================================================ +# Main Collector Class +# ============================================================================ + +class HeatmapCollector(StackTraceCollector): + """Collector that generates coverage.py-style heatmap HTML output with line intensity. + + This collector creates detailed HTML reports showing which lines of code + were executed most frequently during profiling, similar to coverage.py + but showing execution "heat" rather than just coverage. + """ + + # File naming and formatting constants + FILE_INDEX_FORMAT = "file_{:04d}.html" + + def __init__(self, *args, **kwargs): + """Initialize the heatmap collector with data structures for analysis.""" + super().__init__(*args, **kwargs) + + # Sample counting data structures + self.line_samples = collections.Counter() + self.file_samples = collections.defaultdict(collections.Counter) + self.line_self_samples = collections.Counter() + self.file_self_samples = collections.defaultdict(collections.Counter) + + # Call graph data structures for navigation + self.call_graph = collections.defaultdict(list) + self.callers_graph = collections.defaultdict(list) + self.function_definitions = {} + + # Edge counting for call path analysis + self.edge_samples = collections.Counter() + + # Statistics and metadata + self._total_samples = 0 + self._path_info = get_python_path_info() + self.stats = {} + + # Color gradient configuration + self._color_gradient = ColorGradient() + + # Template loader (loads all templates once) + self._template_loader = _TemplateLoader() + + # File index (populated during export) + self.file_index = {} + + @property + def _color_cache(self): + """Compatibility property for accessing color cache.""" + return self._color_gradient.cache + + def set_stats(self, sample_interval_usec, duration_sec, sample_rate, error_rate=None, missed_samples=None, **kwargs): + """Set profiling statistics to include in heatmap output. + + Args: + sample_interval_usec: Sampling interval in microseconds + duration_sec: Total profiling duration in seconds + sample_rate: Effective sampling rate + error_rate: Optional error rate during profiling + missed_samples: Optional percentage of missed samples + **kwargs: Additional statistics to include + """ + self.stats = { + "sample_interval_usec": sample_interval_usec, + "duration_sec": duration_sec, + "sample_rate": sample_rate, + "error_rate": error_rate, + "missed_samples": missed_samples, + "python_version": sys.version, + "python_implementation": platform.python_implementation(), + "platform": platform.platform(), + } + self.stats.update(kwargs) + + def process_frames(self, frames, thread_id): + """Process stack frames and count samples per line. + + Args: + frames: List of frame tuples (filename, lineno, funcname) + frames[0] is the leaf (top of stack, where execution is) + thread_id: Thread ID for this stack trace + """ + self._total_samples += 1 + + # Count each line in the stack and build call graph + for i, frame_info in enumerate(frames): + filename, lineno, funcname = frame_info + + if not self._is_valid_frame(filename, lineno): + continue + + # frames[0] is the leaf - where execution is actually happening + is_leaf = (i == 0) + self._record_line_sample(filename, lineno, funcname, is_leaf=is_leaf) + + # Build call graph for adjacent frames + if i + 1 < len(frames): + self._record_call_relationship(frames[i], frames[i + 1]) + + def _is_valid_frame(self, filename, lineno): + """Check if a frame should be included in the heatmap.""" + # Skip internal or invalid files + if not filename or filename.startswith('<') or filename.startswith('['): + return False + + # Skip invalid frames with corrupted filename data + if filename == "__init__" and lineno == 0: + return False + + return True + + def _record_line_sample(self, filename, lineno, funcname, is_leaf=False): + """Record a sample for a specific line.""" + # Track cumulative samples (all occurrences in stack) + self.line_samples[(filename, lineno)] += 1 + self.file_samples[filename][lineno] += 1 + + # Track self/leaf samples (only when at top of stack) + if is_leaf: + self.line_self_samples[(filename, lineno)] += 1 + self.file_self_samples[filename][lineno] += 1 + + # Record function definition location + if funcname and (filename, funcname) not in self.function_definitions: + self.function_definitions[(filename, funcname)] = lineno + + def _record_call_relationship(self, callee_frame, caller_frame): + """Record caller/callee relationship between adjacent frames.""" + callee_filename, callee_lineno, callee_funcname = callee_frame + caller_filename, caller_lineno, caller_funcname = caller_frame + + # Skip internal files for call graph + if callee_filename.startswith('<') or callee_filename.startswith('['): + return + + # Get the callee's function definition line + callee_def_line = self.function_definitions.get( + (callee_filename, callee_funcname), callee_lineno + ) + + # Record caller -> callee relationship + caller_key = (caller_filename, caller_lineno) + callee_info = (callee_filename, callee_def_line, callee_funcname) + if callee_info not in self.call_graph[caller_key]: + self.call_graph[caller_key].append(callee_info) + + # Record callee <- caller relationship + callee_key = (callee_filename, callee_def_line) + caller_info = (caller_filename, caller_lineno, caller_funcname) + if caller_info not in self.callers_graph[callee_key]: + self.callers_graph[callee_key].append(caller_info) + + # Count this call edge for path analysis + edge_key = (caller_key, callee_key) + self.edge_samples[edge_key] += 1 + + def export(self, output_path): + """Export heatmap data as HTML files in a directory. + + Args: + output_path: Path where to create the heatmap output directory + """ + if not self.file_samples: + print("Warning: No heatmap data to export") + return + + try: + output_dir = self._prepare_output_directory(output_path) + file_stats = self._calculate_file_stats() + self._create_file_index(file_stats) + + # Generate individual file reports + self._generate_file_reports(output_dir, file_stats) + + # Generate index page + self._generate_index_html(output_dir / 'index.html', file_stats) + + self._print_export_summary(output_dir, file_stats) + + except Exception as e: + print(f"Error: Failed to export heatmap: {e}") + raise + + def _prepare_output_directory(self, output_path): + """Create output directory for heatmap files.""" + output_dir = Path(output_path) + if output_dir.suffix == '.html': + output_dir = output_dir.with_suffix('') + + try: + output_dir.mkdir(exist_ok=True, parents=True) + except (IOError, OSError) as e: + raise RuntimeError(f"Failed to create output directory {output_dir}: {e}") from e + + return output_dir + + def _create_file_index(self, file_stats: List[FileStats]): + """Create mapping from filenames to HTML file names.""" + self.file_index = { + stat.filename: self.FILE_INDEX_FORMAT.format(i) + for i, stat in enumerate(file_stats) + } + + def _generate_file_reports(self, output_dir, file_stats: List[FileStats]): + """Generate HTML report for each source file.""" + for stat in file_stats: + file_path = output_dir / self.file_index[stat.filename] + line_counts = self.file_samples[stat.filename] + valid_line_counts = {line: count for line, count in line_counts.items() if line >= 0} + + self_counts = self.file_self_samples.get(stat.filename, {}) + valid_self_counts = {line: count for line, count in self_counts.items() if line >= 0} + + self._generate_file_html( + file_path, + stat.filename, + valid_line_counts, + valid_self_counts, + stat + ) + + def _print_export_summary(self, output_dir, file_stats: List[FileStats]): + """Print summary of exported heatmap.""" + print(f"Heatmap output written to {output_dir}/") + print(f" - Index: {output_dir / 'index.html'}") + print(f" - {len(file_stats)} source file(s) analyzed") + + def _calculate_file_stats(self) -> List[FileStats]: + """Calculate statistics for each file. + + Returns: + List of FileStats objects sorted by total samples + """ + file_stats = [] + for filename, line_counts in self.file_samples.items(): + # Skip special frames + if filename in ('~', '...', '.') or filename.startswith('<') or filename.startswith('['): + continue + + # Filter out lines with -1 (special frames) + valid_line_counts = {line: count for line, count in line_counts.items() if line >= 0} + if not valid_line_counts: + continue + + # Get self samples for this file + self_line_counts = self.file_self_samples.get(filename, {}) + valid_self_counts = {line: count for line, count in self_line_counts.items() if line >= 0} + + total_samples = sum(valid_line_counts.values()) + total_self_samples = sum(valid_self_counts.values()) + num_lines = len(valid_line_counts) + max_samples = max(valid_line_counts.values()) + max_self_samples = max(valid_self_counts.values()) if valid_self_counts else 0 + module_name, module_type = extract_module_name(filename, self._path_info) + + file_stats.append(FileStats( + filename=filename, + module_name=module_name, + module_type=module_type, + total_samples=total_samples, + total_self_samples=total_self_samples, + num_lines=num_lines, + max_samples=max_samples, + max_self_samples=max_self_samples, + percentage=0.0 + )) + + # Sort by total samples and calculate percentages + file_stats.sort(key=lambda x: x.total_samples, reverse=True) + if file_stats: + max_total = file_stats[0].total_samples + for stat in file_stats: + stat.percentage = (stat.total_samples / max_total * 100) if max_total > 0 else 0 + + return file_stats + + def _generate_index_html(self, index_path: Path, file_stats: List[FileStats]): + """Generate index.html with list of all profiled files.""" + # Build hierarchical tree + tree = _TreeBuilder.build_file_tree(file_stats) + + # Render tree as HTML + renderer = _HtmlRenderer(self.file_index, self._color_gradient, + self._calculate_intensity_color) + sections_html = renderer.render_hierarchical_html(tree) + + # Format error rate and missed samples with bar classes + error_rate = self.stats.get('error_rate') + if error_rate is not None: + error_rate_str = f"{error_rate:.1f}%" + error_rate_width = min(error_rate, 100) + # Determine bar color class based on rate + if error_rate < 5: + error_rate_class = "good" + elif error_rate < 15: + error_rate_class = "warning" + else: + error_rate_class = "error" + else: + error_rate_str = "N/A" + error_rate_width = 0 + error_rate_class = "good" + + missed_samples = self.stats.get('missed_samples') + if missed_samples is not None: + missed_samples_str = f"{missed_samples:.1f}%" + missed_samples_width = min(missed_samples, 100) + if missed_samples < 5: + missed_samples_class = "good" + elif missed_samples < 15: + missed_samples_class = "warning" + else: + missed_samples_class = "error" + else: + missed_samples_str = "N/A" + missed_samples_width = 0 + missed_samples_class = "good" + + # Populate template + replacements = { + "": f"", + "": f"", + "": self._template_loader.logo_html, + "": str(len(file_stats)), + "": f"{self._total_samples:,}", + "": f"{self.stats.get('duration_sec', 0):.1f}s", + "": f"{self.stats.get('sample_rate', 0):.1f}", + "": error_rate_str, + "": str(error_rate_width), + "": error_rate_class, + "": missed_samples_str, + "": str(missed_samples_width), + "": missed_samples_class, + "": sections_html, + } + + html_content = self._template_loader.index_template + for placeholder, value in replacements.items(): + html_content = html_content.replace(placeholder, value) + + try: + index_path.write_text(html_content, encoding='utf-8') + except (IOError, OSError) as e: + raise RuntimeError(f"Failed to write index file {index_path}: {e}") from e + + def _calculate_intensity_color(self, intensity: float) -> Tuple[int, int, int, float]: + """Calculate RGB color and alpha for given intensity (0-1 range). + + Returns (r, g, b, alpha) tuple representing the heatmap color gradient: + blue -> green -> yellow -> orange -> red + + Results are cached to improve performance. + """ + # Round to 3 decimal places for cache key + cache_key = round(intensity, 3) + if cache_key in self._color_gradient.cache: + return self._color_gradient.cache[cache_key] + + gradient = self._color_gradient + m = gradient.multiplier + + # Color stops with (threshold, rgb_func, alpha_func) + stops = [ + (gradient.stop_1, + lambda i: (0, int(150 * i * m), 255), + lambda i: gradient.alpha_very_cold), + (gradient.stop_2, + lambda i: (0, 255, int(255 * (1 - (i - gradient.stop_1) * m))), + lambda i: gradient.alpha_cold), + (gradient.stop_3, + lambda i: (int(255 * (i - gradient.stop_2) * m), 255, 0), + lambda i: gradient.alpha_medium), + (gradient.stop_4, + lambda i: (255, int(200 - 100 * (i - gradient.stop_3) * m), 0), + lambda i: gradient.alpha_warm), + (gradient.stop_5, + lambda i: (255, int(100 * (1 - (i - gradient.stop_4) * m)), 0), + lambda i: gradient.alpha_hot_base + gradient.alpha_hot_range * (i - gradient.stop_4) * m), + ] + + result = None + for threshold, rgb_func, alpha_func in stops: + if intensity < threshold or threshold == gradient.stop_5: + r, g, b = rgb_func(intensity) + result = (r, g, b, alpha_func(intensity)) + break + + # Fallback + if result is None: + result = (255, 0, 0, 0.75) + + # Cache the result + self._color_gradient.cache[cache_key] = result + return result + + def _generate_file_html(self, output_path: Path, filename: str, + line_counts: Dict[int, int], self_counts: Dict[int, int], + file_stat: FileStats): + """Generate HTML for a single source file with heatmap coloring.""" + # Read source file + try: + source_lines = Path(filename).read_text(encoding='utf-8', errors='replace').splitlines() + except (IOError, OSError) as e: + if not (filename.startswith('<') or filename.startswith('[') or + filename in ('~', '...', '.') or len(filename) < 2): + print(f"Warning: Could not read source file {filename}: {e}") + source_lines = [f"# Source file not available: {filename}"] + + # Generate HTML for each line + max_samples = max(line_counts.values()) if line_counts else 1 + max_self_samples = max(self_counts.values()) if self_counts else 1 + code_lines_html = [ + self._build_line_html(line_num, line_content, line_counts, self_counts, + max_samples, max_self_samples, filename) + for line_num, line_content in enumerate(source_lines, start=1) + ] + + # Populate template + replacements = { + "": html.escape(filename), + "": f"{file_stat.total_samples:,}", + "": f"{file_stat.total_self_samples:,}", + "": str(file_stat.num_lines), + "": f"{file_stat.percentage:.2f}", + "": str(file_stat.max_samples), + "": str(file_stat.max_self_samples), + "": ''.join(code_lines_html), + "": f"", + "": f"", + } + + html_content = self._template_loader.file_template + for placeholder, value in replacements.items(): + html_content = html_content.replace(placeholder, value) + + try: + output_path.write_text(html_content, encoding='utf-8') + except (IOError, OSError) as e: + raise RuntimeError(f"Failed to write file {output_path}: {e}") from e + + def _build_line_html(self, line_num: int, line_content: str, + line_counts: Dict[int, int], self_counts: Dict[int, int], + max_samples: int, max_self_samples: int, filename: str) -> str: + """Build HTML for a single line of source code.""" + cumulative_samples = line_counts.get(line_num, 0) + self_samples = self_counts.get(line_num, 0) + + # Calculate colors for both self and cumulative modes + if cumulative_samples > 0: + cumulative_intensity = cumulative_samples / max_samples if max_samples > 0 else 0 + self_intensity = self_samples / max_self_samples if max_self_samples > 0 and self_samples > 0 else 0 + + # Default to self-based coloring + intensity = self_intensity if self_samples > 0 else cumulative_intensity + r, g, b, alpha = self._calculate_intensity_color(intensity) + bg_color = f"rgba({r}, {g}, {b}, {alpha})" + + # Pre-calculate colors for both modes (for JS toggle) + self_bg_color = self._format_color_for_intensity(self_intensity) if self_samples > 0 else "transparent" + cumulative_bg_color = self._format_color_for_intensity(cumulative_intensity) + + self_display = f"{self_samples:,}" if self_samples > 0 else "" + cumulative_display = f"{cumulative_samples:,}" + tooltip = f"Self: {self_samples:,}, Total: {cumulative_samples:,}" + else: + bg_color = "transparent" + self_bg_color = "transparent" + cumulative_bg_color = "transparent" + self_display = "" + cumulative_display = "" + tooltip = "" + + # Get navigation buttons + nav_buttons_html = self._build_navigation_buttons(filename, line_num) + + # Build line HTML + line_html = html.escape(line_content.rstrip('\n')) + title_attr = f' title="{html.escape(tooltip)}"' if tooltip else "" + + return ( + f'
\n' + f'
{line_num}
\n' + f'
{self_display}
\n' + f'
{cumulative_display}
\n' + f'
{line_html}
\n' + f' {nav_buttons_html}\n' + f'
\n' + ) + + def _format_color_for_intensity(self, intensity: float) -> str: + """Format color as rgba() string for given intensity.""" + r, g, b, alpha = self._calculate_intensity_color(intensity) + return f"rgba({r}, {g}, {b}, {alpha})" + + def _build_navigation_buttons(self, filename: str, line_num: int) -> str: + """Build navigation buttons for callers/callees.""" + line_key = (filename, line_num) + caller_list = self._deduplicate_by_function(self.callers_graph.get(line_key, [])) + callee_list = self._deduplicate_by_function(self.call_graph.get(line_key, [])) + + # Get edge counts for each caller/callee + callers_with_counts = self._get_edge_counts(line_key, caller_list, is_caller=True) + callees_with_counts = self._get_edge_counts(line_key, callee_list, is_caller=False) + + # Build navigation buttons with counts + caller_btn = self._create_navigation_button(callers_with_counts, 'caller', '▲') + callee_btn = self._create_navigation_button(callees_with_counts, 'callee', '▼') + + if caller_btn or callee_btn: + return f'
{caller_btn}{callee_btn}
' + return '' + + def _get_edge_counts(self, line_key: Tuple[str, int], + items: List[Tuple[str, int, str]], + is_caller: bool) -> List[Tuple[str, int, str, int]]: + """Get sample counts for each caller/callee edge.""" + result = [] + for file, line, func in items: + edge_line_key = (file, line) + if is_caller: + edge_key = (edge_line_key, line_key) + else: + edge_key = (line_key, edge_line_key) + + count = self.edge_samples.get(edge_key, 0) + result.append((file, line, func, count)) + + result.sort(key=lambda x: x[3], reverse=True) + return result + + def _deduplicate_by_function(self, items: List[Tuple[str, int, str]]) -> List[Tuple[str, int, str]]: + """Remove duplicate entries based on (file, function) key.""" + seen = {} + result = [] + for file, line, func in items: + key = (file, func) + if key not in seen: + seen[key] = True + result.append((file, line, func)) + return result + + def _create_navigation_button(self, items_with_counts: List[Tuple[str, int, str, int]], + btn_class: str, arrow: str) -> str: + """Create HTML for a navigation button with sample counts.""" + # Filter valid items + valid_items = [(f, l, fn, cnt) for f, l, fn, cnt in items_with_counts + if f in self.file_index and l > 0] + if not valid_items: + return "" + + if len(valid_items) == 1: + file, line, func, count = valid_items[0] + target_html = self.file_index[file] + nav_data = json.dumps({'link': f"{target_html}#line-{line}", 'func': func}) + title = f"Go to {btn_class}: {html.escape(func)} ({count:,} samples)" + return f'' + + # Multiple items - create menu + total_samples = sum(cnt for _, _, _, cnt in valid_items) + items_data = [ + { + 'file': os.path.basename(file), + 'func': func, + 'count': count, + 'link': f"{self.file_index[file]}#line-{line}" + } + for file, line, func, count in valid_items + ] + items_json = html.escape(json.dumps(items_data)) + title = f"{len(items_data)} {btn_class}s ({total_samples:,} samples)" + return f'' diff --git a/Lib/profiling/sampling/sample.py b/Lib/profiling/sampling/sample.py index 88d9a4fa13baf9..46fc1a05afaa74 100644 --- a/Lib/profiling/sampling/sample.py +++ b/Lib/profiling/sampling/sample.py @@ -10,6 +10,7 @@ from .pstats_collector import PstatsCollector from .stack_collector import CollapsedStackCollector, FlamegraphCollector +from .heatmap_collector import HeatmapCollector from .gecko_collector import GeckoCollector from .constants import ( PROFILING_MODE_WALL, @@ -25,7 +26,6 @@ _FREE_THREADED_BUILD = sysconfig.get_config_var("Py_GIL_DISABLED") is not None - class SampleProfiler: def __init__(self, pid, sample_interval_usec, all_threads, *, mode=PROFILING_MODE_WALL, native=False, gc=True, skip_non_matching_threads=True): self.pid = pid diff --git a/Lib/profiling/sampling/stack_collector.py b/Lib/profiling/sampling/stack_collector.py index 146a058a03ac14..e26536093130d1 100644 --- a/Lib/profiling/sampling/stack_collector.py +++ b/Lib/profiling/sampling/stack_collector.py @@ -6,6 +6,7 @@ import linecache import os +from ._css_utils import get_combined_css from .collector import Collector from .string_table import StringTable @@ -331,9 +332,9 @@ def _create_flamegraph_html(self, data): fg_js_path = d3_flame_graph_dir / "d3-flamegraph.min.js" fg_tooltip_js_path = d3_flame_graph_dir / "d3-flamegraph-tooltip.min.js" - html_template = (template_dir / "flamegraph_template.html").read_text(encoding="utf-8") - css_content = (template_dir / "flamegraph.css").read_text(encoding="utf-8") - js_content = (template_dir / "flamegraph.js").read_text(encoding="utf-8") + html_template = (template_dir / "_flamegraph_assets" / "flamegraph_template.html").read_text(encoding="utf-8") + css_content = get_combined_css("flamegraph") + js_content = (template_dir / "_flamegraph_assets" / "flamegraph.js").read_text(encoding="utf-8") # Inline first-party CSS/JS html_template = html_template.replace( diff --git a/Lib/test/test_profiling/test_heatmap.py b/Lib/test/test_profiling/test_heatmap.py new file mode 100644 index 00000000000000..a6ff3b83ea1e0b --- /dev/null +++ b/Lib/test/test_profiling/test_heatmap.py @@ -0,0 +1,653 @@ +"""Tests for the heatmap collector (profiling.sampling).""" + +import os +import shutil +import tempfile +import unittest +from pathlib import Path + +from profiling.sampling.heatmap_collector import ( + HeatmapCollector, + get_python_path_info, + extract_module_name, +) + +from test.support import captured_stdout, captured_stderr + + +# ============================================================================= +# Unit Tests for Public Helper Functions +# ============================================================================= + +class TestPathInfoFunctions(unittest.TestCase): + """Test public helper functions for path information.""" + + def test_get_python_path_info_returns_dict(self): + """Test that get_python_path_info returns a dictionary with expected keys.""" + path_info = get_python_path_info() + + self.assertIsInstance(path_info, dict) + self.assertIn('stdlib', path_info) + self.assertIn('site_packages', path_info) + self.assertIn('sys_path', path_info) + + def test_get_python_path_info_stdlib_is_path_or_none(self): + """Test that stdlib is either a Path object or None.""" + path_info = get_python_path_info() + + if path_info['stdlib'] is not None: + self.assertIsInstance(path_info['stdlib'], Path) + + def test_get_python_path_info_site_packages_is_list(self): + """Test that site_packages is a list.""" + path_info = get_python_path_info() + + self.assertIsInstance(path_info['site_packages'], list) + for item in path_info['site_packages']: + self.assertIsInstance(item, Path) + + def test_get_python_path_info_sys_path_is_list(self): + """Test that sys_path is a list of Path objects.""" + path_info = get_python_path_info() + + self.assertIsInstance(path_info['sys_path'], list) + for item in path_info['sys_path']: + self.assertIsInstance(item, Path) + + def test_extract_module_name_with_none(self): + """Test extract_module_name with None filename.""" + path_info = get_python_path_info() + module_name, module_type = extract_module_name(None, path_info) + + self.assertEqual(module_name, 'unknown') + self.assertEqual(module_type, 'other') + + def test_extract_module_name_with_empty_string(self): + """Test extract_module_name with empty filename.""" + path_info = get_python_path_info() + module_name, module_type = extract_module_name('', path_info) + + self.assertEqual(module_name, 'unknown') + self.assertEqual(module_type, 'other') + + def test_extract_module_name_with_stdlib_file(self): + """Test extract_module_name with a standard library file.""" + path_info = get_python_path_info() + + # Use os module as a known stdlib file + if path_info['stdlib']: + stdlib_file = str(path_info['stdlib'] / 'os.py') + module_name, module_type = extract_module_name(stdlib_file, path_info) + + self.assertEqual(module_type, 'stdlib') + self.assertIn('os', module_name) + + def test_extract_module_name_with_project_file(self): + """Test extract_module_name with a project file.""" + path_info = get_python_path_info() + + # Create a mock project file path + if path_info['sys_path']: + # Use current directory as project path + project_file = '/some/project/path/mymodule.py' + module_name, module_type = extract_module_name(project_file, path_info) + + # Should classify as 'other' if not in sys.path + self.assertIn(module_type, ['project', 'other']) + + def test_extract_module_name_removes_py_extension(self): + """Test that .py extension is removed from module names.""" + path_info = get_python_path_info() + + # Test with a simple .py file + module_name, module_type = extract_module_name('/path/to/test.py', path_info) + + # Module name should not contain .py + self.assertNotIn('.py', module_name) + + def test_extract_module_name_with_special_files(self): + """Test extract_module_name with special filenames like .""" + path_info = get_python_path_info() + + special_files = ['', '', '[eval]'] + for special_file in special_files: + module_name, module_type = extract_module_name(special_file, path_info) + self.assertEqual(module_type, 'other') + + +# ============================================================================= +# Unit Tests for HeatmapCollector Public API +# ============================================================================= + +class TestHeatmapCollectorInit(unittest.TestCase): + """Test HeatmapCollector initialization.""" + + def test_init_creates_empty_data_structures(self): + """Test that __init__ creates empty data structures.""" + collector = HeatmapCollector(sample_interval_usec=100) + + # Check that data structures are initialized + self.assertIsInstance(collector.line_samples, dict) + self.assertIsInstance(collector.file_samples, dict) + self.assertIsInstance(collector.line_self_samples, dict) + self.assertIsInstance(collector.file_self_samples, dict) + self.assertIsInstance(collector.call_graph, dict) + self.assertIsInstance(collector.callers_graph, dict) + self.assertIsInstance(collector.function_definitions, dict) + self.assertIsInstance(collector.edge_samples, dict) + + # Check that they're empty + self.assertEqual(len(collector.line_samples), 0) + self.assertEqual(len(collector.file_samples), 0) + self.assertEqual(len(collector.line_self_samples), 0) + self.assertEqual(len(collector.file_self_samples), 0) + + def test_init_sets_total_samples_to_zero(self): + """Test that total samples starts at zero.""" + collector = HeatmapCollector(sample_interval_usec=100) + self.assertEqual(collector._total_samples, 0) + + def test_init_creates_color_cache(self): + """Test that color cache is initialized.""" + collector = HeatmapCollector(sample_interval_usec=100) + self.assertIsInstance(collector._color_cache, dict) + self.assertEqual(len(collector._color_cache), 0) + + def test_init_gets_path_info(self): + """Test that path info is retrieved during init.""" + collector = HeatmapCollector(sample_interval_usec=100) + self.assertIsNotNone(collector._path_info) + self.assertIn('stdlib', collector._path_info) + + +class TestHeatmapCollectorSetStats(unittest.TestCase): + """Test HeatmapCollector.set_stats() method.""" + + def test_set_stats_stores_all_parameters(self): + """Test that set_stats stores all provided parameters.""" + collector = HeatmapCollector(sample_interval_usec=100) + + collector.set_stats( + sample_interval_usec=500, + duration_sec=10.5, + sample_rate=99.5, + error_rate=0.5 + ) + + self.assertEqual(collector.stats['sample_interval_usec'], 500) + self.assertEqual(collector.stats['duration_sec'], 10.5) + self.assertEqual(collector.stats['sample_rate'], 99.5) + self.assertEqual(collector.stats['error_rate'], 0.5) + + def test_set_stats_includes_system_info(self): + """Test that set_stats includes Python and platform info.""" + collector = HeatmapCollector(sample_interval_usec=100) + collector.set_stats(sample_interval_usec=100, duration_sec=1.0, sample_rate=100.0) + + self.assertIn('python_version', collector.stats) + self.assertIn('python_implementation', collector.stats) + self.assertIn('platform', collector.stats) + + def test_set_stats_accepts_kwargs(self): + """Test that set_stats accepts additional kwargs.""" + collector = HeatmapCollector(sample_interval_usec=100) + + collector.set_stats( + sample_interval_usec=100, + duration_sec=1.0, + sample_rate=100.0, + custom_key='custom_value', + another_key=42 + ) + + self.assertEqual(collector.stats['custom_key'], 'custom_value') + self.assertEqual(collector.stats['another_key'], 42) + + def test_set_stats_with_none_error_rate(self): + """Test set_stats with error_rate=None.""" + collector = HeatmapCollector(sample_interval_usec=100) + collector.set_stats(sample_interval_usec=100, duration_sec=1.0, sample_rate=100.0) + + self.assertIn('error_rate', collector.stats) + self.assertIsNone(collector.stats['error_rate']) + + +class TestHeatmapCollectorProcessFrames(unittest.TestCase): + """Test HeatmapCollector.process_frames() method.""" + + def test_process_frames_increments_total_samples(self): + """Test that process_frames increments total samples count.""" + collector = HeatmapCollector(sample_interval_usec=100) + + initial_count = collector._total_samples + frames = [('file.py', 10, 'func')] + collector.process_frames(frames, thread_id=1) + + self.assertEqual(collector._total_samples, initial_count + 1) + + def test_process_frames_records_line_samples(self): + """Test that process_frames records line samples.""" + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [('test.py', 5, 'test_func')] + collector.process_frames(frames, thread_id=1) + + # Check that line was recorded + self.assertIn(('test.py', 5), collector.line_samples) + self.assertEqual(collector.line_samples[('test.py', 5)], 1) + + def test_process_frames_records_multiple_lines_in_stack(self): + """Test that process_frames records all lines in a stack.""" + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [ + ('file1.py', 10, 'func1'), + ('file2.py', 20, 'func2'), + ('file3.py', 30, 'func3') + ] + collector.process_frames(frames, thread_id=1) + + # All frames should be recorded + self.assertIn(('file1.py', 10), collector.line_samples) + self.assertIn(('file2.py', 20), collector.line_samples) + self.assertIn(('file3.py', 30), collector.line_samples) + + def test_process_frames_distinguishes_self_samples(self): + """Test that process_frames distinguishes self (leaf) samples.""" + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [ + ('leaf.py', 5, 'leaf_func'), # This is the leaf (top of stack) + ('caller.py', 10, 'caller_func') + ] + collector.process_frames(frames, thread_id=1) + + # Leaf should have self sample + self.assertIn(('leaf.py', 5), collector.line_self_samples) + self.assertEqual(collector.line_self_samples[('leaf.py', 5)], 1) + + # Caller should NOT have self sample + self.assertNotIn(('caller.py', 10), collector.line_self_samples) + + def test_process_frames_accumulates_samples(self): + """Test that multiple calls accumulate samples.""" + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [('file.py', 10, 'func')] + + collector.process_frames(frames, thread_id=1) + collector.process_frames(frames, thread_id=1) + collector.process_frames(frames, thread_id=1) + + self.assertEqual(collector.line_samples[('file.py', 10)], 3) + self.assertEqual(collector._total_samples, 3) + + def test_process_frames_ignores_invalid_frames(self): + """Test that process_frames ignores invalid frames.""" + collector = HeatmapCollector(sample_interval_usec=100) + + # These should be ignored + invalid_frames = [ + ('', 1, 'test'), + ('[eval]', 1, 'test'), + ('', 1, 'test'), + (None, 1, 'test'), + ('__init__', 0, 'test'), # Special invalid frame + ] + + for frame in invalid_frames: + collector.process_frames([frame], thread_id=1) + + # Should not record these invalid frames + for frame in invalid_frames: + if frame[0]: + self.assertNotIn((frame[0], frame[1]), collector.line_samples) + + def test_process_frames_builds_call_graph(self): + """Test that process_frames builds call graph relationships.""" + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [ + ('callee.py', 5, 'callee_func'), + ('caller.py', 10, 'caller_func') + ] + collector.process_frames(frames, thread_id=1) + + # Check that call relationship was recorded + caller_key = ('caller.py', 10) + self.assertIn(caller_key, collector.call_graph) + + # Check callers graph + callee_key = ('callee.py', 5) + self.assertIn(callee_key, collector.callers_graph) + + def test_process_frames_records_function_definitions(self): + """Test that process_frames records function definition locations.""" + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [('module.py', 42, 'my_function')] + collector.process_frames(frames, thread_id=1) + + self.assertIn(('module.py', 'my_function'), collector.function_definitions) + self.assertEqual(collector.function_definitions[('module.py', 'my_function')], 42) + + def test_process_frames_tracks_edge_samples(self): + """Test that process_frames tracks edge sample counts.""" + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [ + ('callee.py', 5, 'callee'), + ('caller.py', 10, 'caller') + ] + + # Process same call stack multiple times + collector.process_frames(frames, thread_id=1) + collector.process_frames(frames, thread_id=1) + + # Check that edge count is tracked + self.assertGreater(len(collector.edge_samples), 0) + + def test_process_frames_handles_empty_frames(self): + """Test that process_frames handles empty frame list.""" + collector = HeatmapCollector(sample_interval_usec=100) + + initial_count = collector._total_samples + collector.process_frames([], thread_id=1) + + # Should still increment total samples + self.assertEqual(collector._total_samples, initial_count + 1) + + def test_process_frames_with_file_samples_dict(self): + """Test that file_samples dict is properly populated.""" + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [('test.py', 10, 'func')] + collector.process_frames(frames, thread_id=1) + + self.assertIn('test.py', collector.file_samples) + self.assertIn(10, collector.file_samples['test.py']) + self.assertEqual(collector.file_samples['test.py'][10], 1) + + +class TestHeatmapCollectorExport(unittest.TestCase): + """Test HeatmapCollector.export() method.""" + + def setUp(self): + """Set up test directory.""" + self.test_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.test_dir) + + def test_export_creates_output_directory(self): + """Test that export creates the output directory.""" + collector = HeatmapCollector(sample_interval_usec=100) + + # Add some data + frames = [('test.py', 10, 'func')] + collector.process_frames(frames, thread_id=1) + + output_path = os.path.join(self.test_dir, 'heatmap_output') + + with captured_stdout(), captured_stderr(): + collector.export(output_path) + + self.assertTrue(os.path.exists(output_path)) + self.assertTrue(os.path.isdir(output_path)) + + def test_export_creates_index_html(self): + """Test that export creates index.html.""" + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [('test.py', 10, 'func')] + collector.process_frames(frames, thread_id=1) + + output_path = os.path.join(self.test_dir, 'heatmap_output') + + with captured_stdout(), captured_stderr(): + collector.export(output_path) + + index_path = os.path.join(output_path, 'index.html') + self.assertTrue(os.path.exists(index_path)) + + def test_export_creates_file_htmls(self): + """Test that export creates individual file HTMLs.""" + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [('test.py', 10, 'func')] + collector.process_frames(frames, thread_id=1) + + output_path = os.path.join(self.test_dir, 'heatmap_output') + + with captured_stdout(), captured_stderr(): + collector.export(output_path) + + # Check for file_XXXX.html files + html_files = [f for f in os.listdir(output_path) + if f.startswith('file_') and f.endswith('.html')] + self.assertGreater(len(html_files), 0) + + def test_export_with_empty_data(self): + """Test export with no data collected.""" + collector = HeatmapCollector(sample_interval_usec=100) + + output_path = os.path.join(self.test_dir, 'empty_output') + + # Should handle empty data gracefully + with captured_stdout(), captured_stderr(): + collector.export(output_path) + + def test_export_handles_html_suffix(self): + """Test that export handles .html suffix in output path.""" + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [('test.py', 10, 'func')] + collector.process_frames(frames, thread_id=1) + + # Path with .html suffix should be stripped + output_path = os.path.join(self.test_dir, 'output.html') + + with captured_stdout(), captured_stderr(): + collector.export(output_path) + + # Should create directory without .html + expected_dir = os.path.join(self.test_dir, 'output') + self.assertTrue(os.path.exists(expected_dir)) + + def test_export_with_multiple_files(self): + """Test export with multiple files.""" + collector = HeatmapCollector(sample_interval_usec=100) + + # Add samples for multiple files + collector.process_frames([('file1.py', 10, 'func1')], thread_id=1) + collector.process_frames([('file2.py', 20, 'func2')], thread_id=1) + collector.process_frames([('file3.py', 30, 'func3')], thread_id=1) + + output_path = os.path.join(self.test_dir, 'multi_file') + + with captured_stdout(), captured_stderr(): + collector.export(output_path) + + # Should create HTML for each file + html_files = [f for f in os.listdir(output_path) + if f.startswith('file_') and f.endswith('.html')] + self.assertGreaterEqual(len(html_files), 3) + + def test_export_index_contains_file_references(self): + """Test that index.html contains references to profiled files.""" + collector = HeatmapCollector(sample_interval_usec=100) + collector.set_stats(sample_interval_usec=100, duration_sec=1.0, sample_rate=100.0) + + frames = [('mytest.py', 10, 'my_func')] + collector.process_frames(frames, thread_id=1) + + output_path = os.path.join(self.test_dir, 'test_output') + + with captured_stdout(), captured_stderr(): + collector.export(output_path) + + index_path = os.path.join(output_path, 'index.html') + with open(index_path, 'r', encoding='utf-8') as f: + content = f.read() + + # Should contain reference to the file + self.assertIn('mytest', content) + + def test_export_file_html_has_line_numbers(self): + """Test that exported file HTML contains line numbers.""" + collector = HeatmapCollector(sample_interval_usec=100) + + # Create a temporary Python file + temp_file = os.path.join(self.test_dir, 'temp_source.py') + with open(temp_file, 'w') as f: + f.write('def test():\n pass\n') + + frames = [(temp_file, 1, 'test')] + collector.process_frames(frames, thread_id=1) + + output_path = os.path.join(self.test_dir, 'line_test') + + with captured_stdout(), captured_stderr(): + collector.export(output_path) + + # Find the generated file HTML + html_files = [f for f in os.listdir(output_path) + if f.startswith('file_') and f.endswith('.html')] + + if html_files: + with open(os.path.join(output_path, html_files[0]), 'r', encoding='utf-8') as f: + content = f.read() + + # Should have line-related content + self.assertIn('line-', content) + + +class MockFrameInfo: + """Mock FrameInfo for testing since the real one isn't accessible.""" + + def __init__(self, filename, lineno, funcname): + self.filename = filename + self.lineno = lineno + self.funcname = funcname + + def __repr__(self): + return f"MockFrameInfo(filename='{self.filename}', lineno={self.lineno}, funcname='{self.funcname}')" + + +class MockThreadInfo: + """Mock ThreadInfo for testing since the real one isn't accessible.""" + + def __init__(self, thread_id, frame_info): + self.thread_id = thread_id + self.frame_info = frame_info + + def __repr__(self): + return f"MockThreadInfo(thread_id={self.thread_id}, frame_info={self.frame_info})" + + +class MockInterpreterInfo: + """Mock InterpreterInfo for testing since the real one isn't accessible.""" + + def __init__(self, interpreter_id, threads): + self.interpreter_id = interpreter_id + self.threads = threads + + def __repr__(self): + return f"MockInterpreterInfo(interpreter_id={self.interpreter_id}, threads={self.threads})" + + +class TestHeatmapCollector(unittest.TestCase): + """Tests for HeatmapCollector functionality.""" + + def test_heatmap_collector_basic(self): + """Test basic HeatmapCollector functionality.""" + collector = HeatmapCollector(sample_interval_usec=100) + + # Test empty state + self.assertEqual(len(collector.file_samples), 0) + self.assertEqual(len(collector.line_samples), 0) + + # Test collecting sample data + test_frames = [ + MockInterpreterInfo( + 0, + [MockThreadInfo( + 1, + [("file.py", 10, "func1"), ("file.py", 20, "func2")], + )] + ) + ] + collector.collect(test_frames) + + # Should have recorded samples for the file + self.assertGreater(len(collector.line_samples), 0) + self.assertIn("file.py", collector.file_samples) + + # Check that line samples were recorded + file_data = collector.file_samples["file.py"] + self.assertGreater(len(file_data), 0) + + def test_heatmap_collector_export(self): + """Test heatmap HTML export functionality.""" + heatmap_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, heatmap_dir) + + collector = HeatmapCollector(sample_interval_usec=100) + + # Create test data with multiple files + test_frames1 = [ + MockInterpreterInfo( + 0, + [MockThreadInfo(1, [("file.py", 10, "func1"), ("file.py", 20, "func2")])], + ) + ] + test_frames2 = [ + MockInterpreterInfo( + 0, + [MockThreadInfo(1, [("file.py", 10, "func1"), ("file.py", 20, "func2")])], + ) + ] # Same stack + test_frames3 = [ + MockInterpreterInfo(0, [MockThreadInfo(1, [("other.py", 5, "other_func")])]) + ] + + collector.collect(test_frames1) + collector.collect(test_frames2) + collector.collect(test_frames3) + + # Export heatmap + with (captured_stdout(), captured_stderr()): + collector.export(heatmap_dir) + + # Verify index.html was created + index_path = os.path.join(heatmap_dir, "index.html") + self.assertTrue(os.path.exists(index_path)) + self.assertGreater(os.path.getsize(index_path), 0) + + # Check index contains HTML content + with open(index_path, "r", encoding="utf-8") as f: + content = f.read() + + # Should be valid HTML + self.assertIn("", content.lower()) + self.assertIn(" Date: Wed, 3 Dec 2025 01:16:37 -0600 Subject: [PATCH 242/638] gh-142145: Remove quadratic behavior in node ID cache clearing (GH-142146) * Remove quadratic behavior in node ID cache clearing Co-authored-by: Jacob Walls <38668450+jacobtylerwalls@users.noreply.github.com> * Add news fragment --------- Co-authored-by: Jacob Walls <38668450+jacobtylerwalls@users.noreply.github.com> --- Lib/test/test_minidom.py | 18 ++++++++++++++++++ Lib/xml/dom/minidom.py | 9 +-------- ...5-12-01-09-36-45.gh-issue-142145.tcAUhg.rst | 1 + 3 files changed, 20 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2025-12-01-09-36-45.gh-issue-142145.tcAUhg.rst diff --git a/Lib/test/test_minidom.py b/Lib/test/test_minidom.py index 4f25e9c2a03cb4..4fa5a4e6768b25 100644 --- a/Lib/test/test_minidom.py +++ b/Lib/test/test_minidom.py @@ -2,6 +2,7 @@ import copy import pickle +import time import io from test import support import unittest @@ -173,6 +174,23 @@ def testAppendChild(self): self.assertEqual(dom.documentElement.childNodes[-1].data, "Hello") dom.unlink() + def testAppendChildNoQuadraticComplexity(self): + impl = getDOMImplementation() + + newdoc = impl.createDocument(None, "some_tag", None) + top_element = newdoc.documentElement + children = [newdoc.createElement(f"child-{i}") for i in range(1, 2 ** 15 + 1)] + element = top_element + + start = time.time() + for child in children: + element.appendChild(child) + element = child + end = time.time() + + # This example used to take at least 30 seconds. + self.assertLess(end - start, 1) + def testAppendChildFragment(self): dom, orig, c1, c2, c3, frag = self._create_fragment_test_nodes() dom.documentElement.appendChild(frag) diff --git a/Lib/xml/dom/minidom.py b/Lib/xml/dom/minidom.py index db51f350ea0153..0a2ccc00f1857d 100644 --- a/Lib/xml/dom/minidom.py +++ b/Lib/xml/dom/minidom.py @@ -292,13 +292,6 @@ def _append_child(self, node): childNodes.append(node) node.parentNode = self -def _in_document(node): - # return True iff node is part of a document tree - while node is not None: - if node.nodeType == Node.DOCUMENT_NODE: - return True - node = node.parentNode - return False def _write_data(writer, text, attr): "Writes datachars to writer." @@ -1555,7 +1548,7 @@ def _clear_id_cache(node): if node.nodeType == Node.DOCUMENT_NODE: node._id_cache.clear() node._id_search_stack = None - elif _in_document(node): + elif node.ownerDocument: node.ownerDocument._id_cache.clear() node.ownerDocument._id_search_stack= None diff --git a/Misc/NEWS.d/next/Security/2025-12-01-09-36-45.gh-issue-142145.tcAUhg.rst b/Misc/NEWS.d/next/Security/2025-12-01-09-36-45.gh-issue-142145.tcAUhg.rst new file mode 100644 index 00000000000000..440bc7794c69ef --- /dev/null +++ b/Misc/NEWS.d/next/Security/2025-12-01-09-36-45.gh-issue-142145.tcAUhg.rst @@ -0,0 +1 @@ +Remove quadratic behavior in ``xml.minidom`` node ID cache clearing. From 88cd5d9850d2dc51abe43eb84198904d9870c26e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartosz=20S=C5=82awecki?= Date: Wed, 3 Dec 2025 10:11:40 +0100 Subject: [PATCH 243/638] gh-142170: Add pymanager link to issue template menu (#142199) Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- .github/ISSUE_TEMPLATE/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 75d174307ce160..de6e8756b03d80 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -5,3 +5,6 @@ contact_links: - name: "Proposing new features" about: "Submit major feature proposal (e.g. syntax changes) to an ideas forum first." url: "https://discuss.python.org/c/ideas/6" + - name: "Python Install Manager issues" + about: "Report issues with the Python Install Manager (for Windows)" + url: "https://github.com/python/pymanager/issues" From 4172644d78d58189e46424af0aea302b1d78e2de Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 3 Dec 2025 13:59:14 +0100 Subject: [PATCH 244/638] gh-142206: multiprocessing.resource_tracker: Decode messages using older protocol (GH-142215) --- Lib/multiprocessing/resource_tracker.py | 63 +++++++++++++------ Lib/test/_test_multiprocessing.py | 26 +++++++- ...-12-03-09-36-29.gh-issue-142206.ilwegH.rst | 4 ++ 3 files changed, 73 insertions(+), 20 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-12-03-09-36-29.gh-issue-142206.ilwegH.rst diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index b0f9099f4a59f3..3606d1effb495b 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -68,6 +68,13 @@ def __init__(self): self._exitcode = None self._reentrant_messages = deque() + # True to use colon-separated lines, rather than JSON lines, + # for internal communication. (Mainly for testing). + # Filenames not supported by the simple format will always be sent + # using JSON. + # The reader should understand all formats. + self._use_simple_format = False + def _reentrant_call_error(self): # gh-109629: this happens if an explicit call to the ResourceTracker # gets interrupted by a garbage collection, invoking a finalizer (*) @@ -200,7 +207,9 @@ def _launch(self): os.close(r) def _make_probe_message(self): - """Return a JSON-encoded probe message.""" + """Return a probe message.""" + if self._use_simple_format: + return b'PROBE:0:noop\n' return ( json.dumps( {"cmd": "PROBE", "rtype": "noop"}, @@ -267,6 +276,15 @@ def _write(self, msg): assert nbytes == len(msg), f"{nbytes=} != {len(msg)=}" def _send(self, cmd, name, rtype): + if self._use_simple_format and '\n' not in name: + msg = f"{cmd}:{name}:{rtype}\n".encode("ascii") + if len(msg) > 512: + # posix guarantees that writes to a pipe of less than PIPE_BUF + # bytes are atomic, and that PIPE_BUF >= 512 + raise ValueError('msg too long') + self._ensure_running_and_write(msg) + return + # POSIX guarantees that writes to a pipe of less than PIPE_BUF (512 on Linux) # bytes are atomic. Therefore, we want the message to be shorter than 512 bytes. # POSIX shm_open() and sem_open() require the name, including its leading slash, @@ -286,6 +304,7 @@ def _send(self, cmd, name, rtype): # The entire JSON message is guaranteed < PIPE_BUF (512 bytes) by construction. assert len(msg) <= 512, f"internal error: message too long ({len(msg)} bytes)" + assert msg.startswith(b'{') self._ensure_running_and_write(msg) @@ -296,6 +315,30 @@ def _send(self, cmd, name, rtype): getfd = _resource_tracker.getfd +def _decode_message(line): + if line.startswith(b'{'): + try: + obj = json.loads(line.decode('ascii')) + except Exception as e: + raise ValueError("malformed resource_tracker message: %r" % (line,)) from e + + cmd = obj["cmd"] + rtype = obj["rtype"] + b64 = obj.get("base64_name", "") + + if not isinstance(cmd, str) or not isinstance(rtype, str) or not isinstance(b64, str): + raise ValueError("malformed resource_tracker fields: %r" % (obj,)) + + try: + name = base64.urlsafe_b64decode(b64).decode('utf-8', 'surrogateescape') + except ValueError as e: + raise ValueError("malformed resource_tracker base64_name: %r" % (b64,)) from e + else: + cmd, rest = line.strip().decode('ascii').split(':', maxsplit=1) + name, rtype = rest.rsplit(':', maxsplit=1) + return cmd, rtype, name + + def main(fd): '''Run resource tracker.''' # protect the process from ^C and "killall python" etc @@ -318,23 +361,7 @@ def main(fd): with open(fd, 'rb') as f: for line in f: try: - try: - obj = json.loads(line.decode('ascii')) - except Exception as e: - raise ValueError("malformed resource_tracker message: %r" % (line,)) from e - - cmd = obj["cmd"] - rtype = obj["rtype"] - b64 = obj.get("base64_name", "") - - if not isinstance(cmd, str) or not isinstance(rtype, str) or not isinstance(b64, str): - raise ValueError("malformed resource_tracker fields: %r" % (obj,)) - - try: - name = base64.urlsafe_b64decode(b64).decode('utf-8', 'surrogateescape') - except ValueError as e: - raise ValueError("malformed resource_tracker base64_name: %r" % (b64,)) from e - + cmd, rtype, name = _decode_message(line) cleanup_func = _CLEANUP_FUNCS.get(rtype, None) if cleanup_func is None: raise ValueError( diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index d718a27231897f..d03eb1dfb253ec 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -39,6 +39,7 @@ from test.support import socket_helper from test.support import threading_helper from test.support import warnings_helper +from test.support import subTests from test.support.script_helper import assert_python_failure, assert_python_ok # Skip tests if _multiprocessing wasn't built. @@ -4383,6 +4384,19 @@ def test_copy(self): self.assertEqual(bar.z, 2 ** 33) +def resource_tracker_format_subtests(func): + """Run given test using both resource tracker communication formats""" + def _inner(self, *args, **kwargs): + tracker = resource_tracker._resource_tracker + for use_simple_format in False, True: + with ( + self.subTest(use_simple_format=use_simple_format), + unittest.mock.patch.object( + tracker, '_use_simple_format', use_simple_format) + ): + func(self, *args, **kwargs) + return _inner + @unittest.skipUnless(HAS_SHMEM, "requires multiprocessing.shared_memory") @hashlib_helper.requires_hashdigest('sha256') class _TestSharedMemory(BaseTestCase): @@ -4662,6 +4676,7 @@ def test_shared_memory_SharedMemoryServer_ignores_sigint(self): smm.shutdown() @unittest.skipIf(os.name != "posix", "resource_tracker is posix only") + @resource_tracker_format_subtests def test_shared_memory_SharedMemoryManager_reuses_resource_tracker(self): # bpo-36867: test that a SharedMemoryManager uses the # same resource_tracker process as its parent. @@ -4913,6 +4928,7 @@ def test_shared_memory_cleaned_after_process_termination(self): "shared_memory objects to clean up at shutdown", err) @unittest.skipIf(os.name != "posix", "resource_tracker is posix only") + @resource_tracker_format_subtests def test_shared_memory_untracking(self): # gh-82300: When a separate Python process accesses shared memory # with track=False, it must not cause the memory to be deleted @@ -4940,6 +4956,7 @@ def test_shared_memory_untracking(self): mem.close() @unittest.skipIf(os.name != "posix", "resource_tracker is posix only") + @resource_tracker_format_subtests def test_shared_memory_tracking(self): # gh-82300: When a separate Python process accesses shared memory # with track=True, it must cause the memory to be deleted when @@ -7353,13 +7370,18 @@ def test_forkpty(self): @unittest.skipUnless(HAS_SHMEM, "requires multiprocessing.shared_memory") class TestSharedMemoryNames(unittest.TestCase): - def test_that_shared_memory_name_with_colons_has_no_resource_tracker_errors(self): + @subTests('use_simple_format', (True, False)) + def test_that_shared_memory_name_with_colons_has_no_resource_tracker_errors( + self, use_simple_format): # Test script that creates and cleans up shared memory with colon in name test_script = textwrap.dedent(""" import sys from multiprocessing import shared_memory + from multiprocessing import resource_tracker import time + resource_tracker._resource_tracker._use_simple_format = %s + # Test various patterns of colons in names test_names = [ "a:b", @@ -7387,7 +7409,7 @@ def test_that_shared_memory_name_with_colons_has_no_resource_tracker_errors(self sys.exit(1) print("SUCCESS") - """) + """ % use_simple_format) rc, out, err = assert_python_ok("-c", test_script) self.assertIn(b"SUCCESS", out) diff --git a/Misc/NEWS.d/next/Library/2025-12-03-09-36-29.gh-issue-142206.ilwegH.rst b/Misc/NEWS.d/next/Library/2025-12-03-09-36-29.gh-issue-142206.ilwegH.rst new file mode 100644 index 00000000000000..90e4dd96985979 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-03-09-36-29.gh-issue-142206.ilwegH.rst @@ -0,0 +1,4 @@ +The resource tracker in the :mod:`multiprocessing` module can now understand +messages from older versions of itself. This avoids issues with upgrading +Python while it is running. (Note that such 'in-place' upgrades are not +tested.) From 7e5fcae09bb0e87ed48cb593f7f46d715e48a102 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 3 Dec 2025 14:33:32 +0100 Subject: [PATCH 245/638] gh-142217: Remove internal _Py_Identifier functions (#142219) Remove internal functions: * _PyDict_ContainsId() * _PyDict_DelItemId() * _PyDict_GetItemIdWithError() * _PyDict_SetItemId() * _PyEval_GetBuiltinId() * _PyObject_CallMethodIdNoArgs() * _PyObject_CallMethodIdObjArgs() * _PyObject_CallMethodIdOneArg() * _PyObject_VectorcallMethodId() * _PyUnicode_EqualToASCIIId() These functions were not exported and so no usable outside CPython. --- Include/internal/pycore_call.h | 33 -------------------- Include/internal/pycore_ceval.h | 2 -- Include/internal/pycore_dict.h | 7 ----- Include/internal/pycore_unicodeobject.h | 8 ----- Objects/call.c | 33 -------------------- Objects/dictobject.c | 41 ------------------------- Objects/odictobject.c | 1 - Objects/unicodeobject.c | 41 ------------------------- Python/ceval.c | 6 ---- 9 files changed, 172 deletions(-) diff --git a/Include/internal/pycore_call.h b/Include/internal/pycore_call.h index 506da06f7087d2..4f4cf02f64b828 100644 --- a/Include/internal/pycore_call.h +++ b/Include/internal/pycore_call.h @@ -64,39 +64,6 @@ PyAPI_FUNC(PyObject*) _PyObject_CallMethod( PyObject *name, const char *format, ...); -extern PyObject* _PyObject_CallMethodIdObjArgs( - PyObject *obj, - _Py_Identifier *name, - ...); - -static inline PyObject * -_PyObject_VectorcallMethodId( - _Py_Identifier *name, PyObject *const *args, - size_t nargsf, PyObject *kwnames) -{ - PyObject *oname = _PyUnicode_FromId(name); /* borrowed */ - if (!oname) { - return _Py_NULL; - } - return PyObject_VectorcallMethod(oname, args, nargsf, kwnames); -} - -static inline PyObject * -_PyObject_CallMethodIdNoArgs(PyObject *self, _Py_Identifier *name) -{ - size_t nargsf = 1 | PY_VECTORCALL_ARGUMENTS_OFFSET; - return _PyObject_VectorcallMethodId(name, &self, nargsf, _Py_NULL); -} - -static inline PyObject * -_PyObject_CallMethodIdOneArg(PyObject *self, _Py_Identifier *name, PyObject *arg) -{ - PyObject *args[2] = {self, arg}; - size_t nargsf = 2 | PY_VECTORCALL_ARGUMENTS_OFFSET; - assert(arg != NULL); - return _PyObject_VectorcallMethodId(name, args, nargsf, _Py_NULL); -} - /* === Vectorcall protocol (PEP 590) ============================= */ diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 9d81833a2343f2..762d8ef067e288 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -33,8 +33,6 @@ extern int _PyEval_SetOpcodeTrace(PyFrameObject *f, bool enable); // Export for 'array' shared extension PyAPI_FUNC(PyObject*) _PyEval_GetBuiltin(PyObject *); -extern PyObject* _PyEval_GetBuiltinId(_Py_Identifier *); - extern void _PyEval_SetSwitchInterval(unsigned long microseconds); extern unsigned long _PyEval_GetSwitchInterval(void); diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index b8fe360321d14b..1193f496da132d 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -36,13 +36,6 @@ extern int _PyDict_DelItem_KnownHash_LockHeld(PyObject *mp, PyObject *key, extern int _PyDict_Contains_KnownHash(PyObject *, PyObject *, Py_hash_t); -// "Id" variants -extern PyObject* _PyDict_GetItemIdWithError(PyObject *dp, - _Py_Identifier *key); -extern int _PyDict_ContainsId(PyObject *, _Py_Identifier *); -extern int _PyDict_SetItemId(PyObject *dp, _Py_Identifier *key, PyObject *item); -extern int _PyDict_DelItemId(PyObject *mp, _Py_Identifier *key); - extern int _PyDict_Next( PyObject *mp, Py_ssize_t *pos, PyObject **key, PyObject **value, Py_hash_t *hash); diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index e7ca65a56b6ec3..97dda73f9b584d 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -307,14 +307,6 @@ PyAPI_FUNC(PyObject*) _PyUnicode_JoinArray( Py_ssize_t seqlen ); -/* Test whether a unicode is equal to ASCII identifier. Return 1 if true, - 0 otherwise. The right argument must be ASCII identifier. - Any error occurs inside will be cleared before return. */ -extern int _PyUnicode_EqualToASCIIId( - PyObject *left, /* Left string */ - _Py_Identifier *right /* Right identifier */ - ); - // Test whether a unicode is equal to ASCII string. Return 1 if true, // 0 otherwise. The right argument must be ASCII-encoded string. // Any error occurs inside will be cleared before return. diff --git a/Objects/call.c b/Objects/call.c index bd8617825b585e..c69015abfb3ed5 100644 --- a/Objects/call.c +++ b/Objects/call.c @@ -891,39 +891,6 @@ PyObject_CallMethodObjArgs(PyObject *obj, PyObject *name, ...) } -PyObject * -_PyObject_CallMethodIdObjArgs(PyObject *obj, _Py_Identifier *name, ...) -{ - PyThreadState *tstate = _PyThreadState_GET(); - if (obj == NULL || name == NULL) { - return null_error(tstate); - } - - PyObject *oname = _PyUnicode_FromId(name); /* borrowed */ - if (!oname) { - return NULL; - } - _PyCStackRef method; - _PyThreadState_PushCStackRef(tstate, &method); - int is_method = _PyObject_GetMethodStackRef(tstate, obj, oname, &method.ref); - if (PyStackRef_IsNull(method.ref)) { - _PyThreadState_PopCStackRef(tstate, &method); - return NULL; - } - PyObject *callable = PyStackRef_AsPyObjectBorrow(method.ref); - - obj = is_method ? obj : NULL; - - va_list vargs; - va_start(vargs, name); - PyObject *result = object_vacall(tstate, obj, callable, vargs); - va_end(vargs); - - _PyThreadState_PopCStackRef(tstate, &method); - return result; -} - - PyObject * PyObject_CallFunctionObjArgs(PyObject *callable, ...) { diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 14de21f3c67210..ee1c173ae4abb0 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -2527,18 +2527,6 @@ _PyDict_GetItemWithError(PyObject *dp, PyObject *kv) return _PyDict_GetItem_KnownHash(dp, kv, hash); // borrowed reference } -PyObject * -_PyDict_GetItemIdWithError(PyObject *dp, _Py_Identifier *key) -{ - PyObject *kv; - kv = _PyUnicode_FromId(key); /* borrowed */ - if (kv == NULL) - return NULL; - Py_hash_t hash = unicode_get_hash(kv); - assert (hash != -1); /* interned strings have their hash value initialised */ - return _PyDict_GetItem_KnownHash(dp, kv, hash); // borrowed reference -} - PyObject * _PyDict_GetItemStringWithError(PyObject *v, const char *key) { @@ -4845,16 +4833,6 @@ _PyDict_Contains_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) return 0; } -int -_PyDict_ContainsId(PyObject *op, _Py_Identifier *key) -{ - PyObject *kv = _PyUnicode_FromId(key); /* borrowed */ - if (kv == NULL) { - return -1; - } - return PyDict_Contains(op, kv); -} - /* Hack to implement "key in dict" */ static PySequenceMethods dict_as_sequence = { 0, /* sq_length */ @@ -5035,16 +5013,6 @@ PyDict_GetItemStringRef(PyObject *v, const char *key, PyObject **result) return res; } -int -_PyDict_SetItemId(PyObject *v, _Py_Identifier *key, PyObject *item) -{ - PyObject *kv; - kv = _PyUnicode_FromId(key); /* borrowed */ - if (kv == NULL) - return -1; - return PyDict_SetItem(v, kv, item); -} - int PyDict_SetItemString(PyObject *v, const char *key, PyObject *item) { @@ -5060,15 +5028,6 @@ PyDict_SetItemString(PyObject *v, const char *key, PyObject *item) return err; } -int -_PyDict_DelItemId(PyObject *v, _Py_Identifier *key) -{ - PyObject *kv = _PyUnicode_FromId(key); /* borrowed */ - if (kv == NULL) - return -1; - return PyDict_DelItem(v, kv); -} - int PyDict_DelItemString(PyObject *v, const char *key) { diff --git a/Objects/odictobject.c b/Objects/odictobject.c index 45d2ea0203a9ff..25928028919c9c 100644 --- a/Objects/odictobject.c +++ b/Objects/odictobject.c @@ -223,7 +223,6 @@ PyDict_DelItem PyMapping_DelItem PyDict_DelItemString PyMapping_DelItemString PyDict_GetItem - PyDict_GetItemWithError PyObject_GetItem -_PyDict_GetItemIdWithError - PyDict_GetItemString PyMapping_GetItemString PyDict_Items PyMapping_Items PyDict_Keys PyMapping_Keys diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 7f9f75126a9e56..f737a885f197a0 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -11194,47 +11194,6 @@ _PyUnicode_EqualToASCIIString(PyObject *unicode, const char *str) memcmp(PyUnicode_1BYTE_DATA(unicode), str, len) == 0; } -int -_PyUnicode_EqualToASCIIId(PyObject *left, _Py_Identifier *right) -{ - PyObject *right_uni; - - assert(_PyUnicode_CHECK(left)); - assert(right->string); -#ifndef NDEBUG - for (const char *p = right->string; *p; p++) { - assert((unsigned char)*p < 128); - } -#endif - - if (!PyUnicode_IS_ASCII(left)) - return 0; - - right_uni = _PyUnicode_FromId(right); /* borrowed */ - if (right_uni == NULL) { - /* memory error or bad data */ - PyErr_Clear(); - return _PyUnicode_EqualToASCIIString(left, right->string); - } - - if (left == right_uni) - return 1; - - assert(PyUnicode_CHECK_INTERNED(right_uni)); - if (PyUnicode_CHECK_INTERNED(left)) { - return 0; - } - - Py_hash_t right_hash = PyUnicode_HASH(right_uni); - assert(right_hash != -1); - Py_hash_t hash = PyUnicode_HASH(left); - if (hash != -1 && hash != right_hash) { - return 0; - } - - return unicode_eq(left, right_uni); -} - PyObject * PyUnicode_RichCompare(PyObject *left, PyObject *right, int op) { diff --git a/Python/ceval.c b/Python/ceval.c index 5381cd826dfd19..39fb38b7307814 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2828,12 +2828,6 @@ _PyEval_GetBuiltin(PyObject *name) return attr; } -PyObject * -_PyEval_GetBuiltinId(_Py_Identifier *name) -{ - return _PyEval_GetBuiltin(_PyUnicode_FromId(name)); -} - PyObject * PyEval_GetLocals(void) { From f6f456f95092142c4b6d038b839975bf7db4d1f2 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Wed, 3 Dec 2025 15:24:17 +0100 Subject: [PATCH 246/638] gh-142038: Expand guard for types_world_is_stopped() to fix debug builds without assertions (#142039) --- Objects/typeobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 4c6ff51493f799..cbe0215359e29d 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -81,7 +81,7 @@ class object "PyObject *" "&PyBaseObject_Type" #define END_TYPE_DICT_LOCK() Py_END_CRITICAL_SECTION2() -#ifndef NDEBUG +#if !defined(NDEBUG) || defined(Py_DEBUG) // Return true if the world is currently stopped. static bool types_world_is_stopped(void) From aea5531583aaa8bfdf3ebca914e9c694617c3489 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 3 Dec 2025 16:14:53 +0100 Subject: [PATCH 247/638] gh-135676: Reword the f-string (and t-string) section (GH-137469) Much of the information was duplicated in stdtypes.rst; this PR keeps lexical/syntactical details in Lexical Analysis and the evaluation & runtime behaviour in Standard types, with cross-references between the two. Since the t-string section only listed differences from f-strings, and the grammar for the two is equivalent, that section was moved to Standard types almost entirely. Co-authored-by: Blaise Pabon Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/library/stdtypes.rst | 196 ++++++++------- Doc/reference/expressions.rst | 2 +- Doc/reference/lexical_analysis.rst | 377 +++++++++++++++++------------ 3 files changed, 332 insertions(+), 243 deletions(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 8b896011734df5..086da1a705c30f 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -2656,6 +2656,8 @@ expression support in the :mod:`re` module). single: : (colon); in formatted string literal single: = (equals); for help in debugging using string literals +.. _stdtypes-fstrings: + Formatted String Literals (f-strings) ------------------------------------- @@ -2664,123 +2666,147 @@ Formatted String Literals (f-strings) The :keyword:`await` and :keyword:`async for` can be used in expressions within f-strings. .. versionchanged:: 3.8 - Added the debugging operator (``=``) + Added the debug specifier (``=``) .. versionchanged:: 3.12 Many restrictions on expressions within f-strings have been removed. Notably, nested strings, comments, and backslashes are now permitted. An :dfn:`f-string` (formally a :dfn:`formatted string literal`) is a string literal that is prefixed with ``f`` or ``F``. -This type of string literal allows embedding arbitrary Python expressions -within *replacement fields*, which are delimited by curly brackets (``{}``). -These expressions are evaluated at runtime, similarly to :meth:`str.format`, -and are converted into regular :class:`str` objects. -For example: +This type of string literal allows embedding the results of arbitrary Python +expressions within *replacement fields*, which are delimited by curly +brackets (``{}``). +Each replacement field must contain an expression, optionally followed by: -.. doctest:: +* a *debug specifier* -- an equal sign (``=``); +* a *conversion specifier* -- ``!s``, ``!r`` or ``!a``; and/or +* a *format specifier* prefixed with a colon (``:``). - >>> who = 'nobody' - >>> nationality = 'Spanish' - >>> f'{who.title()} expects the {nationality} Inquisition!' - 'Nobody expects the Spanish Inquisition!' +See the :ref:`Lexical Analysis section on f-strings ` for details +on the syntax of these fields. -It is also possible to use a multi line f-string: +Debug specifier +^^^^^^^^^^^^^^^ -.. doctest:: +.. versionadded:: 3.8 - >>> f'''This is a string - ... on two lines''' - 'This is a string\non two lines' +If a debug specifier -- an equal sign (``=``) -- appears after the replacement +field expression, the resulting f-string will contain the expression's source, +the equal sign, and the value of the expression. +This is often useful for debugging:: -A single opening curly bracket, ``'{'``, marks a *replacement field* that -can contain any Python expression: + >>> number = 14.3 + >>> f'{number=}' + 'number=14.3' -.. doctest:: - - >>> nationality = 'Spanish' - >>> f'The {nationality} Inquisition!' - 'The Spanish Inquisition!' +Whitespace before, inside and after the expression, as well as whitespace +after the equal sign, is significant --- it is retained in the result:: -To include a literal ``{`` or ``}``, use a double bracket: + >>> f'{ number - 4 = }' + ' number - 4 = 10.3' -.. doctest:: - >>> x = 42 - >>> f'{{x}} is {x}' - '{x} is 42' +Conversion specifier +^^^^^^^^^^^^^^^^^^^^ -Functions can also be used, and :ref:`format specifiers `: - -.. doctest:: - - >>> from math import sqrt - >>> f'√2 \N{ALMOST EQUAL TO} {sqrt(2):.5f}' - '√2 ≈ 1.41421' - -Any non-string expression is converted using :func:`str`, by default: - -.. doctest:: +By default, the value of a replacement field expression is converted to +a string using :func:`str`:: >>> from fractions import Fraction - >>> f'{Fraction(1, 3)}' + >>> one_third = Fraction(1, 3) + >>> f'{one_third}' '1/3' -To use an explicit conversion, use the ``!`` (exclamation mark) operator, -followed by any of the valid formats, which are: +When a debug specifier but no format specifier is used, the default conversion +instead uses :func:`repr`:: -========== ============== -Conversion Meaning -========== ============== -``!a`` :func:`ascii` -``!r`` :func:`repr` -``!s`` :func:`str` -========== ============== + >>> f'{one_third = }' + 'one_third = Fraction(1, 3)' -For example: +The conversion can be specified explicitly using one of these specifiers: -.. doctest:: +* ``!s`` for :func:`str` +* ``!r`` for :func:`repr` +* ``!a`` for :func:`ascii` - >>> from fractions import Fraction - >>> f'{Fraction(1, 3)!s}' +For example:: + + >>> str(one_third) '1/3' - >>> f'{Fraction(1, 3)!r}' + >>> repr(one_third) 'Fraction(1, 3)' - >>> question = '¿Dónde está el Presidente?' - >>> print(f'{question!a}') - '\xbfD\xf3nde est\xe1 el Presidente?' - -While debugging it may be helpful to see both the expression and its value, -by using the equals sign (``=``) after the expression. -This preserves spaces within the brackets, and can be used with a converter. -By default, the debugging operator uses the :func:`repr` (``!r``) conversion. -For example: -.. doctest:: + >>> f'{one_third!s} is {one_third!r}' + '1/3 is Fraction(1, 3)' - >>> from fractions import Fraction - >>> calculation = Fraction(1, 3) - >>> f'{calculation=}' - 'calculation=Fraction(1, 3)' - >>> f'{calculation = }' - 'calculation = Fraction(1, 3)' - >>> f'{calculation = !s}' - 'calculation = 1/3' - -Once the output has been evaluated, it can be formatted using a -:ref:`format specifier ` following a colon (``':'``). -After the expression has been evaluated, and possibly converted to a string, -the :meth:`!__format__` method of the result is called with the format specifier, -or the empty string if no format specifier is given. -The formatted result is then used as the final value for the replacement field. -For example: + >>> string = "¡kočka 😸!" + >>> ascii(string) + "'\\xa1ko\\u010dka \\U0001f638!'" -.. doctest:: + >>> f'{string = !a}' + "string = '\\xa1ko\\u010dka \\U0001f638!'" + + +Format specifier +^^^^^^^^^^^^^^^^ + +After the expression has been evaluated, and possibly converted using an +explicit conversion specifier, it is formatted using the :func:`format` function. +If the replacement field includes a *format specifier* introduced by a colon +(``:``), the specifier is passed to :func:`!format` as the second argument. +The result of :func:`!format` is then used as the final value for the +replacement field. For example:: >>> from fractions import Fraction - >>> f'{Fraction(1, 7):.6f}' - '0.142857' - >>> f'{Fraction(1, 7):_^+10}' - '___+1/7___' + >>> one_third = Fraction(1, 3) + >>> f'{one_third:.6f}' + '0.333333' + >>> f'{one_third:_^+10}' + '___+1/3___' + >>> >>> f'{one_third!r:_^20}' + '___Fraction(1, 3)___' + >>> f'{one_third = :~>10}~' + 'one_third = ~~~~~~~1/3~' + +.. _stdtypes-tstrings: + +Template String Literals (t-strings) +------------------------------------ + +An :dfn:`t-string` (formally a :dfn:`template string literal`) is +a string literal that is prefixed with ``t`` or ``T``. + +These strings follow the same syntax and evaluation rules as +:ref:`formatted string literals `, +with for the following differences: + +* Rather than evaluating to a ``str`` object, template string literals evaluate + to a :class:`string.templatelib.Template` object. + +* The :func:`format` protocol is not used. + Instead, the format specifier and conversions (if any) are passed to + a new :class:`~string.templatelib.Interpolation` object that is created + for each evaluated expression. + It is up to code that processes the resulting :class:`~string.templatelib.Template` + object to decide how to handle format specifiers and conversions. + +* Format specifiers containing nested replacement fields are evaluated eagerly, + prior to being passed to the :class:`~string.templatelib.Interpolation` object. + For instance, an interpolation of the form ``{amount:.{precision}f}`` will + evaluate the inner expression ``{precision}`` to determine the value of the + ``format_spec`` attribute. + If ``precision`` were to be ``2``, the resulting format specifier + would be ``'.2f'``. + +* When the equals sign ``'='`` is provided in an interpolation expression, + the text of the expression is appended to the literal string that precedes + the relevant interpolation. + This includes the equals sign and any surrounding whitespace. + The :class:`!Interpolation` instance for the expression will be created as + normal, except that :attr:`~string.templatelib.Interpolation.conversion` will + be set to '``r``' (:func:`repr`) by default. + If an explicit conversion or format specifier are provided, + this will override the default behaviour. .. _old-string-formatting: diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index c655d6c52ecc16..165dfa69f880d0 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -174,7 +174,7 @@ Formally: .. grammar-snippet:: :group: python-grammar - strings: ( `STRING` | fstring)+ | tstring+ + strings: ( `STRING` | `fstring`)+ | `tstring`+ This feature is defined at the syntactical level, so it only works with literals. To concatenate string expressions at run time, the '+' operator may be used:: diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index 129dc10d07f7c9..9322d8571f7ab6 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -345,7 +345,15 @@ Whitespace between tokens Except at the beginning of a logical line or in string literals, the whitespace characters space, tab and formfeed can be used interchangeably to separate -tokens. Whitespace is needed between two tokens only if their concatenation +tokens: + +.. grammar-snippet:: + :group: python-grammar + + whitespace: ' ' | tab | formfeed + + +Whitespace is needed between two tokens only if their concatenation could otherwise be interpreted as a different token. For example, ``ab`` is one token, but ``a b`` is two tokens. However, ``+a`` and ``+ a`` both produce two tokens, ``+`` and ``a``, as ``+a`` is not a valid token. @@ -1032,124 +1040,59 @@ f-strings --------- .. versionadded:: 3.6 +.. versionchanged:: 3.7 + The :keyword:`await` and :keyword:`async for` can be used in expressions + within f-strings. +.. versionchanged:: 3.8 + Added the debug specifier (``=``) +.. versionchanged:: 3.12 + Many restrictions on expressions within f-strings have been removed. + Notably, nested strings, comments, and backslashes are now permitted. A :dfn:`formatted string literal` or :dfn:`f-string` is a string literal -that is prefixed with '``f``' or '``F``'. These strings may contain -replacement fields, which are expressions delimited by curly braces ``{}``. -While other string literals always have a constant value, formatted strings -are really expressions evaluated at run time. - -Escape sequences are decoded like in ordinary string literals (except when -a literal is also marked as a raw string). After decoding, the grammar -for the contents of the string is: - -.. productionlist:: python-grammar - f_string: (`literal_char` | "{{" | "}}" | `replacement_field`)* - replacement_field: "{" `f_expression` ["="] ["!" `conversion`] [":" `format_spec`] "}" - f_expression: (`conditional_expression` | "*" `or_expr`) - : ("," `conditional_expression` | "," "*" `or_expr`)* [","] - : | `yield_expression` - conversion: "s" | "r" | "a" - format_spec: (`literal_char` | `replacement_field`)* - literal_char: - -The parts of the string outside curly braces are treated literally, -except that any doubled curly braces ``'{{'`` or ``'}}'`` are replaced -with the corresponding single curly brace. A single opening curly -bracket ``'{'`` marks a replacement field, which starts with a -Python expression. To display both the expression text and its value after -evaluation, (useful in debugging), an equal sign ``'='`` may be added after the -expression. A conversion field, introduced by an exclamation point ``'!'`` may -follow. A format specifier may also be appended, introduced by a colon ``':'``. -A replacement field ends with a closing curly bracket ``'}'``. +that is prefixed with '``f``' or '``F``'. +Unlike other string literals, f-strings do not have a constant value. +They may contain *replacement fields* delimited by curly braces ``{}``. +Replacement fields contain expressions which are evaluated at run time. +For example:: + + >>> who = 'nobody' + >>> nationality = 'Spanish' + >>> f'{who.title()} expects the {nationality} Inquisition!' + 'Nobody expects the Spanish Inquisition!' + +Any doubled curly braces (``{{`` or ``}}``) outside replacement fields +are replaced with the corresponding single curly brace:: + + >>> print(f'{{...}}') + {...} + +Other characters outside replacement fields are treated like in ordinary +string literals. +This means that escape sequences are decoded (except when a literal is +also marked as a raw string), and newlines are possible in triple-quoted +f-strings:: + + >>> name = 'Galahad' + >>> favorite_color = 'blue' + >>> print(f'{name}:\t{favorite_color}') + Galahad: blue + >>> print(rf"C:\Users\{name}") + C:\Users\Galahad + >>> print(f'''Three shall be the number of the counting + ... and the number of the counting shall be three.''') + Three shall be the number of the counting + and the number of the counting shall be three. Expressions in formatted string literals are treated like regular -Python expressions surrounded by parentheses, with a few exceptions. -An empty expression is not allowed, and both :keyword:`lambda` and -assignment expressions ``:=`` must be surrounded by explicit parentheses. +Python expressions. Each expression is evaluated in the context where the formatted string literal -appears, in order from left to right. Replacement expressions can contain -newlines in both single-quoted and triple-quoted f-strings and they can contain -comments. Everything that comes after a ``#`` inside a replacement field -is a comment (even closing braces and quotes). In that case, replacement fields -must be closed in a different line. - -.. code-block:: text - - >>> f"abc{a # This is a comment }" - ... + 3}" - 'abc5' - -.. versionchanged:: 3.7 - Prior to Python 3.7, an :keyword:`await` expression and comprehensions - containing an :keyword:`async for` clause were illegal in the expressions - in formatted string literals due to a problem with the implementation. - -.. versionchanged:: 3.12 - Prior to Python 3.12, comments were not allowed inside f-string replacement - fields. - -When the equal sign ``'='`` is provided, the output will have the expression -text, the ``'='`` and the evaluated value. Spaces after the opening brace -``'{'``, within the expression and after the ``'='`` are all retained in the -output. By default, the ``'='`` causes the :func:`repr` of the expression to be -provided, unless there is a format specified. When a format is specified it -defaults to the :func:`str` of the expression unless a conversion ``'!r'`` is -declared. - -.. versionadded:: 3.8 - The equal sign ``'='``. - -If a conversion is specified, the result of evaluating the expression -is converted before formatting. Conversion ``'!s'`` calls :func:`str` on -the result, ``'!r'`` calls :func:`repr`, and ``'!a'`` calls :func:`ascii`. - -The result is then formatted using the :func:`format` protocol. The -format specifier is passed to the :meth:`~object.__format__` method of the -expression or conversion result. An empty string is passed when the -format specifier is omitted. The formatted result is then included in -the final value of the whole string. - -Top-level format specifiers may include nested replacement fields. These nested -fields may include their own conversion fields and :ref:`format specifiers -`, but may not include more deeply nested replacement fields. The -:ref:`format specifier mini-language ` is the same as that used by -the :meth:`str.format` method. - -Formatted string literals may be concatenated, but replacement fields -cannot be split across literals. - -Some examples of formatted string literals:: - - >>> name = "Fred" - >>> f"He said his name is {name!r}." - "He said his name is 'Fred'." - >>> f"He said his name is {repr(name)}." # repr() is equivalent to !r - "He said his name is 'Fred'." - >>> width = 10 - >>> precision = 4 - >>> value = decimal.Decimal("12.34567") - >>> f"result: {value:{width}.{precision}}" # nested fields - 'result: 12.35' - >>> today = datetime(year=2017, month=1, day=27) - >>> f"{today:%B %d, %Y}" # using date format specifier - 'January 27, 2017' - >>> f"{today=:%B %d, %Y}" # using date format specifier and debugging - 'today=January 27, 2017' - >>> number = 1024 - >>> f"{number:#0x}" # using integer format specifier - '0x400' - >>> foo = "bar" - >>> f"{ foo = }" # preserves whitespace - " foo = 'bar'" - >>> line = "The mill's closed" - >>> f"{line = }" - 'line = "The mill\'s closed"' - >>> f"{line = :20}" - "line = The mill's closed " - >>> f"{line = !r:20}" - 'line = "The mill\'s closed" ' +appears, in order from left to right. +An empty expression is not allowed, and both :keyword:`lambda` and +assignment expressions ``:=`` must be surrounded by explicit parentheses:: + >>> f'{(half := 1/2)}, {half * 42}' + '0.5, 21.0' Reusing the outer f-string quoting type inside a replacement field is permitted:: @@ -1158,10 +1101,6 @@ permitted:: >>> f"abc {a["x"]} def" 'abc 2 def' -.. versionchanged:: 3.12 - Prior to Python 3.12, reuse of the same quoting type of the outer f-string - inside a replacement field was not possible. - Backslashes are also allowed in replacement fields and are evaluated the same way as in any other context:: @@ -1172,23 +1111,84 @@ way as in any other context:: b c -.. versionchanged:: 3.12 - Prior to Python 3.12, backslashes were not permitted inside an f-string - replacement field. +It is possible to nest f-strings:: + + >>> name = 'world' + >>> f'Repeated:{f' hello {name}' * 3}' + 'Repeated: hello world hello world hello world' + +Portable Python programs should not use more than 5 levels of nesting. + +.. impl-detail:: + + CPython does not limit nesting of f-strings. + +Replacement expressions can contain newlines in both single-quoted and +triple-quoted f-strings and they can contain comments. +Everything that comes after a ``#`` inside a replacement field +is a comment (even closing braces and quotes). +This means that replacement fields with comments must be closed in a +different line: + +.. code-block:: text + + >>> a = 2 + >>> f"abc{a # This comment }" continues until the end of the line + ... + 3}" + 'abc5' + +After the expression, replacement fields may optionally contain: + +* a *debug specifier* -- an equal sign (``=``), optionally surrounded by + whitespace on one or both sides; +* a *conversion specifier* -- ``!s``, ``!r`` or ``!a``; and/or +* a *format specifier* prefixed with a colon (``:``). + +See the :ref:`Standard Library section on f-strings ` +for details on how these fields are evaluated. + +As that section explains, *format specifiers* are passed as the second argument +to the :func:`format` function to format a replacement field value. +For example, they can be used to specify a field width and padding characters +using the :ref:`Format Specification Mini-Language `:: -Formatted string literals cannot be used as docstrings, even if they do not -include expressions. + >>> number = 14.3 + >>> f'{number:20.7f}' + ' 14.3000000' -:: +Top-level format specifiers may include nested replacement fields:: + + >>> field_size = 20 + >>> precision = 7 + >>> f'{number:{field_size}.{precision}f}' + ' 14.3000000' + +These nested fields may include their own conversion fields and +:ref:`format specifiers `:: + + >>> number = 3 + >>> f'{number:{field_size}}' + ' 3' + >>> f'{number:{field_size:05}}' + '00000000000000000003' + +However, these nested fields may not include more deeply nested replacement +fields. + +Formatted string literals cannot be used as :term:`docstrings `, +even if they do not include expressions:: >>> def foo(): ... f"Not a docstring" ... - >>> foo.__doc__ is None - True + >>> print(foo.__doc__) + None + +.. seealso:: -See also :pep:`498` for the proposal that added formatted string literals, -and :meth:`str.format`, which uses a related format string mechanism. + * :pep:`498` -- Literal String Interpolation + * :pep:`701` -- Syntactic formalization of f-strings + * :meth:`str.format`, which uses a related format string mechanism. .. _t-strings: @@ -1201,36 +1201,99 @@ t-strings A :dfn:`template string literal` or :dfn:`t-string` is a string literal that is prefixed with '``t``' or '``T``'. -These strings follow the same syntax and evaluation rules as -:ref:`formatted string literals `, with the following differences: - -* Rather than evaluating to a ``str`` object, template string literals evaluate - to a :class:`string.templatelib.Template` object. - -* The :func:`format` protocol is not used. - Instead, the format specifier and conversions (if any) are passed to - a new :class:`~string.templatelib.Interpolation` object that is created - for each evaluated expression. - It is up to code that processes the resulting :class:`~string.templatelib.Template` - object to decide how to handle format specifiers and conversions. - -* Format specifiers containing nested replacement fields are evaluated eagerly, - prior to being passed to the :class:`~string.templatelib.Interpolation` object. - For instance, an interpolation of the form ``{amount:.{precision}f}`` will - evaluate the inner expression ``{precision}`` to determine the value of the - ``format_spec`` attribute. - If ``precision`` were to be ``2``, the resulting format specifier - would be ``'.2f'``. - -* When the equals sign ``'='`` is provided in an interpolation expression, - the text of the expression is appended to the literal string that precedes - the relevant interpolation. - This includes the equals sign and any surrounding whitespace. - The :class:`!Interpolation` instance for the expression will be created as - normal, except that :attr:`~string.templatelib.Interpolation.conversion` will - be set to '``r``' (:func:`repr`) by default. - If an explicit conversion or format specifier are provided, - this will override the default behaviour. +These strings follow the same syntax rules as +:ref:`formatted string literals `. +For differences in evaluation rules, see the +:ref:`Standard Library section on t-strings ` + + +Formal grammar for f-strings +---------------------------- + +F-strings are handled partly by the :term:`lexical analyzer`, which produces the +tokens :py:data:`~token.FSTRING_START`, :py:data:`~token.FSTRING_MIDDLE` +and :py:data:`~token.FSTRING_END`, and partly by the parser, which handles +expressions in the replacement field. +The exact way the work is split is a CPython implementation detail. + +Correspondingly, the f-string grammar is a mix of +:ref:`lexical and syntactic definitions `. + +Whitespace is significant in these situations: + +* There may be no whitespace in :py:data:`~token.FSTRING_START` (between + the prefix and quote). +* Whitespace in :py:data:`~token.FSTRING_MIDDLE` is part of the literal + string contents. +* In ``fstring_replacement_field``, if ``f_debug_specifier`` is present, + all whitespace after the opening brace until the ``f_debug_specifier``, + as well as whitespace immediatelly following ``f_debug_specifier``, + is retained as part of the expression. + + .. impl-detail:: + + The expression is not handled in the tokenization phase; it is + retrieved from the source code using locations of the ``{`` token + and the token after ``=``. + + +The ``FSTRING_MIDDLE`` definition uses +:ref:`negative lookaheads ` (``!``) +to indicate special characters (backslash, newline, ``{``, ``}``) and +sequences (``f_quote``). + +.. grammar-snippet:: + :group: python-grammar + + fstring: `FSTRING_START` `fstring_middle`* `FSTRING_END` + + FSTRING_START: `fstringprefix` ("'" | '"' | "'''" | '"""') + FSTRING_END: `f_quote` + fstringprefix: <("f" | "fr" | "rf"), case-insensitive> + f_debug_specifier: '=' + f_quote: + + fstring_middle: + | `fstring_replacement_field` + | `FSTRING_MIDDLE` + FSTRING_MIDDLE: + | (!"\" !`newline` !'{' !'}' !`f_quote`) `source_character` + | `stringescapeseq` + | "{{" + | "}}" + | + fstring_replacement_field: + | '{' `f_expression` [`f_debug_specifier`] [`fstring_conversion`] + [`fstring_full_format_spec`] '}' + fstring_conversion: + | "!" ("s" | "r" | "a") + fstring_full_format_spec: + | ':' `fstring_format_spec`* + fstring_format_spec: + | `FSTRING_MIDDLE` + | `fstring_replacement_field` + f_expression: + | ','.(`conditional_expression` | "*" `or_expr`)+ [","] + | `yield_expression` + +.. note:: + + In the above grammar snippet, the ``f_quote`` and ``FSTRING_MIDDLE`` rules + are context-sensitive -- they depend on the contents of ``FSTRING_START`` + of the nearest enclosing ``fstring``. + + Constructing a more traditional formal grammar from this template is left + as an exercise for the reader. + +The grammar for t-strings is identical to the one for f-strings, with *t* +instead of *f* at the beginning of rule and token names and in the prefix. + +.. grammar-snippet:: + :group: python-grammar + + tstring: TSTRING_START tstring_middle* TSTRING_END + + .. _numbers: From 62423c9c36f428ba07c83aeea7cbacc7cbb34ed2 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 3 Dec 2025 17:43:35 +0000 Subject: [PATCH 248/638] GH-141794: Limit size of generated machine code. (GH-142228) * Factor out bodies of the largest uops, to reduce jit code size. * Factor out common assert, also reducing jit code size. * Limit size of jitted code for a single executor to 1MB. --- Include/internal/pycore_ceval.h | 58 ++ Include/internal/pycore_jit.h | 3 + Include/internal/pycore_uop.h | 5 + Lib/test/test_generated_cases.py | 54 +- Python/bytecodes.c | 179 ++--- Python/ceval.c | 284 +++++++- Python/ceval_macros.h | 2 +- Python/executor_cases.c.h | 1094 +++++++++++------------------- Python/generated_cases.c.h | 1078 ++++++++++------------------- Python/jit.c | 4 + Python/optimizer_analysis.c | 30 + Python/optimizer_cases.c.h | 346 +++++----- Tools/cases_generator/stack.py | 2 +- Tools/jit/template.c | 6 + 14 files changed, 1410 insertions(+), 1735 deletions(-) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 762d8ef067e288..6bf33bddd5b877 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -408,6 +408,64 @@ _PyForIter_VirtualIteratorNext(PyThreadState* tstate, struct _PyInterpreterFrame PyAPI_DATA(const _Py_CODEUNIT *) _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR; +/* Helper functions for large uops */ + +PyAPI_FUNC(PyObject *) +_Py_VectorCall_StackRefSteal( + _PyStackRef callable, + _PyStackRef *arguments, + int total_args, + _PyStackRef kwnames); + +PyAPI_FUNC(PyObject *) +_Py_BuiltinCallFast_StackRefSteal( + _PyStackRef callable, + _PyStackRef *arguments, + int total_args); + +PyAPI_FUNC(PyObject *) +_Py_BuiltinCallFastWithKeywords_StackRefSteal( + _PyStackRef callable, + _PyStackRef *arguments, + int total_args); + +PyAPI_FUNC(PyObject *) +_PyCallMethodDescriptorFast_StackRefSteal( + _PyStackRef callable, + PyMethodDef *meth, + PyObject *self, + _PyStackRef *arguments, + int total_args); + +PyAPI_FUNC(PyObject *) +_PyCallMethodDescriptorFastWithKeywords_StackRefSteal( + _PyStackRef callable, + PyMethodDef *meth, + PyObject *self, + _PyStackRef *arguments, + int total_args); + +PyAPI_FUNC(PyObject *) +_Py_CallBuiltinClass_StackRefSteal( + _PyStackRef callable, + _PyStackRef *arguments, + int total_args); + +PyAPI_FUNC(PyObject *) +_Py_BuildString_StackRefSteal( + _PyStackRef *arguments, + int total_args); + +PyAPI_FUNC(PyObject *) +_Py_BuildMap_StackRefSteal( + _PyStackRef *arguments, + int half_args); + +PyAPI_FUNC(void) +_Py_assert_within_stack_bounds( + _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + const char *filename, int lineno); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_jit.h b/Include/internal/pycore_jit.h index 8a88cbf607ba4b..b1550a6ddcfc0b 100644 --- a/Include/internal/pycore_jit.h +++ b/Include/internal/pycore_jit.h @@ -13,6 +13,9 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +/* To be able to reason about code layout and branches, keep code size below 1 MB */ +#define PY_MAX_JIT_CODE_SIZE ((1 << 20)-1) + #ifdef _Py_JIT typedef _Py_CODEUNIT *(*jit_func)(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate); diff --git a/Include/internal/pycore_uop.h b/Include/internal/pycore_uop.h index 4e1b15af42caa3..70576046385675 100644 --- a/Include/internal/pycore_uop.h +++ b/Include/internal/pycore_uop.h @@ -36,7 +36,12 @@ typedef struct _PyUOpInstruction{ } _PyUOpInstruction; // This is the length of the trace we translate initially. +#ifdef Py_DEBUG + // With asserts, the stencils are a lot larger +#define UOP_MAX_TRACE_LENGTH 1000 +#else #define UOP_MAX_TRACE_LENGTH 3000 +#endif #define UOP_BUFFER_SIZE (UOP_MAX_TRACE_LENGTH * sizeof(_PyUOpInstruction)) /* Bloom filter with m = 256 diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 09ce329bdcd14d..ac62e11c274fab 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -165,7 +165,7 @@ def test_inst_one_pop(self): value = stack_pointer[-1]; SPAM(value); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -190,7 +190,7 @@ def test_inst_one_push(self): res = SPAM(); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -247,7 +247,7 @@ def test_binary_op(self): res = SPAM(left, right); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -366,14 +366,14 @@ def test_sync_sp(self): _PyStackRef res; arg = stack_pointer[-1]; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); escaping_call(); stack_pointer = _PyFrame_GetStackPointer(frame); res = Py_None; stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -489,7 +489,7 @@ def test_error_if_pop(self): res = 0; stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -523,7 +523,7 @@ def test_error_if_pop_with_result(self): } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -553,7 +553,7 @@ def test_cache_effect(self): uint32_t extra = read_u32(&this_instr[2].cache); (void)extra; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -640,7 +640,7 @@ def test_macro_instruction(self): } stack_pointer[-3] = res; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -688,7 +688,7 @@ def test_macro_instruction(self): stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer[-3] = res; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -827,7 +827,7 @@ def test_array_input(self): below = stack_pointer[-2 - oparg*2]; SPAM(values, oparg); stack_pointer += -2 - oparg*2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -860,7 +860,7 @@ def test_array_output(self): stack_pointer[-2] = below; stack_pointer[-1 + oparg*3] = above; stack_pointer += oparg*3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -889,7 +889,7 @@ def test_array_input_output(self): above = 0; stack_pointer[0] = above; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -918,11 +918,11 @@ def test_array_error_if(self): extra = stack_pointer[-1 - oparg]; if (oparg == 0) { stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -960,7 +960,7 @@ def test_macro_push_push(self): stack_pointer[0] = val1; stack_pointer[1] = val2; stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -1263,13 +1263,13 @@ def test_flush(self): stack_pointer[0] = a; stack_pointer[1] = b; stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); // SECOND { USE(a, b); } stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -1325,7 +1325,7 @@ def test_pop_on_error_peeks(self): } } stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -1368,14 +1368,14 @@ def test_push_then_error(self): stack_pointer[0] = a; stack_pointer[1] = b; stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } } stack_pointer[0] = a; stack_pointer[1] = b; stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -1661,7 +1661,7 @@ def test_pystackref_frompyobject_new_next_to_cmacro(self): stack_pointer[0] = out1; stack_pointer[1] = out2; stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } """ @@ -1881,7 +1881,7 @@ def test_reassigning_dead_inputs(self): stack_pointer = _PyFrame_GetStackPointer(frame); in = temp; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(in); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2116,7 +2116,7 @@ def test_validate_uop_unused_input(self): output = """ case OP: { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } """ @@ -2133,7 +2133,7 @@ def test_validate_uop_unused_input(self): output = """ case OP: { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } """ @@ -2155,7 +2155,7 @@ def test_validate_uop_unused_output(self): foo = NULL; stack_pointer[0] = foo; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } """ @@ -2173,7 +2173,7 @@ def test_validate_uop_unused_output(self): output = """ case OP: { stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } """ diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 01cd1e8359815a..4ba255d28bdcf6 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1976,14 +1976,8 @@ dummy_func( } inst(BUILD_STRING, (pieces[oparg] -- str)) { - STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); - if (CONVERSION_FAILED(pieces_o)) { - DECREF_INPUTS(); - ERROR_IF(true); - } - PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); - STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); - DECREF_INPUTS(); + PyObject *str_o = _Py_BuildString_StackRefSteal(pieces, oparg); + DEAD(pieces); ERROR_IF(str_o == NULL); str = PyStackRef_FromPyObjectSteal(str_o); } @@ -2098,17 +2092,9 @@ dummy_func( } inst(BUILD_MAP, (values[oparg*2] -- map)) { - STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); - if (CONVERSION_FAILED(values_o)) { - DECREF_INPUTS(); - ERROR_IF(true); - } - PyObject *map_o = _PyDict_FromItems( - values_o, 2, - values_o+1, 2, - oparg); - STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); - DECREF_INPUTS(); + + PyObject *map_o = _Py_BuildMap_StackRefSteal(values, oparg); + DEAD(values); ERROR_IF(map_o == NULL); map = PyStackRef_FromPyObjectStealMortal(map_o); } @@ -3891,27 +3877,20 @@ dummy_func( #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - DECREF_INPUTS(); - ERROR_IF(true); - } - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - DECREF_INPUTS(); + PyObject *res_o = _Py_VectorCall_StackRefSteal( + callable, + arguments, + total_args, + PyStackRef_NULL); + DEAD(args); + DEAD(self_or_null); + DEAD(callable); ERROR_IF(res_o == NULL); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -4186,14 +4165,13 @@ dummy_func( } DEOPT_IF(tp->tp_vectorcall == NULL); STAT_INC(CALL, hit); - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - DECREF_INPUTS(); - ERROR_IF(true); - } - PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - DECREF_INPUTS(); + PyObject *res_o = _Py_CallBuiltinClass_StackRefSteal( + callable, + arguments, + total_args); + DEAD(args); + DEAD(self_or_null); + DEAD(callable); ERROR_IF(res_o == NULL); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -4241,31 +4219,24 @@ dummy_func( op(_CALL_BUILTIN_FAST, (callable, self_or_null, args[oparg] -- res)) { /* Builtin METH_FASTCALL functions, without keywords */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); DEOPT_IF(!PyCFunction_CheckExact(callable_o)); DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL); STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); - /* res = func(self, args, nargs) */ - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - DECREF_INPUTS(); - ERROR_IF(true); - } - PyObject *res_o = _PyCFunctionFast_CAST(cfunc)( - PyCFunction_GET_SELF(callable_o), - args_o, - total_args); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - DECREF_INPUTS(); + PyObject *res_o = _Py_BuiltinCallFast_StackRefSteal( + callable, + arguments, + total_args + ); + DEAD(args); + DEAD(self_or_null); + DEAD(callable); ERROR_IF(res_o == NULL); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -4278,30 +4249,20 @@ dummy_func( op(_CALL_BUILTIN_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- res)) { /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); DEOPT_IF(!PyCFunction_CheckExact(callable_o)); DEOPT_IF(PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)); STAT_INC(CALL, hit); - /* res = func(self, arguments, nargs, kwnames) */ - PyCFunctionFastWithKeywords cfunc = - _PyCFunctionFastWithKeywords_CAST(PyCFunction_GET_FUNCTION(callable_o)); - - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - DECREF_INPUTS(); - ERROR_IF(true); - } - PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - DECREF_INPUTS(); + PyObject *res_o = _Py_BuiltinCallFastWithKeywords_StackRefSteal(callable, arguments, total_args); + DEAD(args); + DEAD(self_or_null); + DEAD(callable); ERROR_IF(res_o == NULL); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -4468,19 +4429,16 @@ dummy_func( assert(self != NULL); EXIT_IF(!Py_IS_TYPE(self, d_type)); STAT_INC(CALL, hit); - int nargs = total_args - 1; - - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - DECREF_INPUTS(); - ERROR_IF(true); - } - PyCFunctionFastWithKeywords cfunc = - _PyCFunctionFastWithKeywords_CAST(meth->ml_meth); - PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - DECREF_INPUTS(); + PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRefSteal( + callable, + meth, + self, + arguments, + total_args + ); + DEAD(args); + DEAD(self_or_null); + DEAD(callable); ERROR_IF(res_o == NULL); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -4548,18 +4506,16 @@ dummy_func( assert(self != NULL); EXIT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); STAT_INC(CALL, hit); - int nargs = total_args - 1; - - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - DECREF_INPUTS(); - ERROR_IF(true); - } - PyCFunctionFast cfunc = _PyCFunctionFast_CAST(meth->ml_meth); - PyObject *res_o = cfunc(self, (args_o + 1), nargs); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - DECREF_INPUTS(); + PyObject *res_o = _PyCallMethodDescriptorFast_StackRefSteal( + callable, + meth, + self, + arguments, + total_args + ); + DEAD(args); + DEAD(self_or_null); + DEAD(callable); ERROR_IF(res_o == NULL); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -4792,30 +4748,21 @@ dummy_func( #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } - /* Callable is not a normal Python function */ - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - DECREF_INPUTS(); - ERROR_IF(true); - } - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames_o); - PyStackRef_CLOSE(kwnames); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - DECREF_INPUTS(); + PyObject *res_o = _Py_VectorCall_StackRefSteal( + callable, + arguments, + total_args, + kwnames); + DEAD(kwnames); + DEAD(args); + DEAD(self_or_null); + DEAD(callable); ERROR_IF(res_o == NULL); res = PyStackRef_FromPyObjectSteal(res_o); } diff --git a/Python/ceval.c b/Python/ceval.c index 39fb38b7307814..1709dda0cbe145 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1017,6 +1017,281 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) #include "ceval_macros.h" + +/* Helper functions to keep the size of the largest uops down */ + +PyObject * +_Py_VectorCall_StackRefSteal( + _PyStackRef callable, + _PyStackRef *arguments, + int total_args, + _PyStackRef kwnames) +{ + PyObject *res; + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + res = NULL; + goto cleanup; + } + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args; + if (kwnames_o != NULL) { + positional_args -= (int)PyTuple_GET_SIZE(kwnames_o); + } + res = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res != NULL) ^ (PyErr_Occurred() != NULL)); +cleanup: + PyStackRef_XCLOSE(kwnames); + // arguments is a pointer into the GC visible stack, + // so we must NULL out values as we clear them. + for (int i = total_args-1; i >= 0; i--) { + _PyStackRef tmp = arguments[i]; + arguments[i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + PyStackRef_CLOSE(callable); + return res; +} + +PyObject * +_Py_BuiltinCallFast_StackRefSteal( + _PyStackRef callable, + _PyStackRef *arguments, + int total_args) +{ + PyObject *res; + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + res = NULL; + goto cleanup; + } + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); + res = _PyCFunctionFast_CAST(cfunc)( + PyCFunction_GET_SELF(callable_o), + args_o, + total_args + ); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res != NULL) ^ (PyErr_Occurred() != NULL)); +cleanup: + // arguments is a pointer into the GC visible stack, + // so we must NULL out values as we clear them. + for (int i = total_args-1; i >= 0; i--) { + _PyStackRef tmp = arguments[i]; + arguments[i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + PyStackRef_CLOSE(callable); + return res; +} + +PyObject * +_Py_BuiltinCallFastWithKeywords_StackRefSteal( + _PyStackRef callable, + _PyStackRef *arguments, + int total_args) +{ + PyObject *res; + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + res = NULL; + goto cleanup; + } + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyCFunctionFastWithKeywords cfunc = + _PyCFunctionFastWithKeywords_CAST(PyCFunction_GET_FUNCTION(callable_o)); + res = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res != NULL) ^ (PyErr_Occurred() != NULL)); +cleanup: + // arguments is a pointer into the GC visible stack, + // so we must NULL out values as we clear them. + for (int i = total_args-1; i >= 0; i--) { + _PyStackRef tmp = arguments[i]; + arguments[i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + PyStackRef_CLOSE(callable); + return res; +} + +PyObject * +_PyCallMethodDescriptorFast_StackRefSteal( + _PyStackRef callable, + PyMethodDef *meth, + PyObject *self, + _PyStackRef *arguments, + int total_args) +{ + PyObject *res; + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + res = NULL; + goto cleanup; + } + assert(((PyMethodDescrObject *)PyStackRef_AsPyObjectBorrow(callable))->d_method == meth); + assert(self == PyStackRef_AsPyObjectBorrow(arguments[0])); + + PyCFunctionFast cfunc = _PyCFunctionFast_CAST(meth->ml_meth); + res = cfunc(self, (args_o + 1), total_args - 1); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res != NULL) ^ (PyErr_Occurred() != NULL)); +cleanup: + // arguments is a pointer into the GC visible stack, + // so we must NULL out values as we clear them. + for (int i = total_args-1; i >= 0; i--) { + _PyStackRef tmp = arguments[i]; + arguments[i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + PyStackRef_CLOSE(callable); + return res; +} + +PyObject * +_PyCallMethodDescriptorFastWithKeywords_StackRefSteal( + _PyStackRef callable, + PyMethodDef *meth, + PyObject *self, + _PyStackRef *arguments, + int total_args) +{ + PyObject *res; + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + res = NULL; + goto cleanup; + } + assert(((PyMethodDescrObject *)PyStackRef_AsPyObjectBorrow(callable))->d_method == meth); + assert(self == PyStackRef_AsPyObjectBorrow(arguments[0])); + + PyCFunctionFastWithKeywords cfunc = + _PyCFunctionFastWithKeywords_CAST(meth->ml_meth); + res = cfunc(self, (args_o + 1), total_args-1, NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res != NULL) ^ (PyErr_Occurred() != NULL)); +cleanup: + // arguments is a pointer into the GC visible stack, + // so we must NULL out values as we clear them. + for (int i = total_args-1; i >= 0; i--) { + _PyStackRef tmp = arguments[i]; + arguments[i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + PyStackRef_CLOSE(callable); + return res; +} + +PyObject * +_Py_CallBuiltinClass_StackRefSteal( + _PyStackRef callable, + _PyStackRef *arguments, + int total_args) +{ + PyObject *res; + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + res = NULL; + goto cleanup; + } + PyTypeObject *tp = (PyTypeObject *)PyStackRef_AsPyObjectBorrow(callable); + res = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res != NULL) ^ (PyErr_Occurred() != NULL)); +cleanup: + // arguments is a pointer into the GC visible stack, + // so we must NULL out values as we clear them. + for (int i = total_args-1; i >= 0; i--) { + _PyStackRef tmp = arguments[i]; + arguments[i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + PyStackRef_CLOSE(callable); + return res; +} + +PyObject * +_Py_BuildString_StackRefSteal( + _PyStackRef *arguments, + int total_args) +{ + PyObject *res; + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + res = NULL; + goto cleanup; + } + res = _PyUnicode_JoinArray(&_Py_STR(empty), args_o, total_args); +cleanup: + // arguments is a pointer into the GC visible stack, + // so we must NULL out values as we clear them. + for (int i = total_args-1; i >= 0; i--) { + _PyStackRef tmp = arguments[i]; + arguments[i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + return res; +} + + + +PyObject * +_Py_BuildMap_StackRefSteal( + _PyStackRef *arguments, + int half_args) +{ + PyObject *res; + STACKREFS_TO_PYOBJECTS(arguments, half_args*2, args_o); + if (CONVERSION_FAILED(args_o)) { + res = NULL; + goto cleanup; + } + res = _PyDict_FromItems( + args_o, 2, + args_o+1, 2, + half_args + ); +cleanup: + // arguments is a pointer into the GC visible stack, + // so we must NULL out values as we clear them. + for (int i = half_args*2-1; i >= 0; i--) { + _PyStackRef tmp = arguments[i]; + arguments[i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + return res; +} + +#ifdef Py_DEBUG +void +_Py_assert_within_stack_bounds( + _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, + const char *filename, int lineno +) { + if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { + return; + } + int level = (int)(stack_pointer - _PyFrame_Stackbase(frame)); + if (level < 0) { + printf("Stack underflow (depth = %d) at %s:%d\n", level, filename, lineno); + fflush(stdout); + abort(); + } + int size = _PyFrame_GetCode(frame)->co_stacksize; + if (level > size) { + printf("Stack overflow (depth = %d) at %s:%d\n", level, filename, lineno); + fflush(stdout); + abort(); + } +} +#endif + int _Py_CheckRecursiveCallPy( PyThreadState *tstate) { @@ -1078,11 +1353,12 @@ _PyObjectArray_FromStackRefArray(_PyStackRef *input, Py_ssize_t nargs, PyObject if (result == NULL) { return NULL; } - result++; } else { result = scratch; } + result++; + result[0] = NULL; /* Keep GCC happy */ for (int i = 0; i < nargs; i++) { result[i] = PyStackRef_AsPyObjectBorrow(input[i]); } @@ -1097,6 +1373,12 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) } } +#ifdef Py_DEBUG +#define ASSERT_WITHIN_STACK_BOUNDS(F, L) _Py_assert_within_stack_bounds(frame, stack_pointer, (F), (L)) +#else +#define ASSERT_WITHIN_STACK_BOUNDS(F, L) (void)0 +#endif + #if _Py_TIER2 // 0 for success, -1 for error. static int diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index c30638c221a03f..edf8fc9a57d74e 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -458,7 +458,7 @@ do { \ #define STACKREFS_TO_PYOBJECTS(ARGS, ARG_COUNT, NAME) \ /* +1 because vectorcall might use -1 to write self */ \ PyObject *NAME##_temp[MAX_STACKREF_SCRATCH+1]; \ - PyObject **NAME = _PyObjectArray_FromStackRefArray(ARGS, ARG_COUNT, NAME##_temp + 1); + PyObject **NAME = _PyObjectArray_FromStackRefArray(ARGS, ARG_COUNT, NAME##_temp); #define STACKREFS_TO_PYOBJECTS_CLEANUP(NAME) \ /* +1 because we +1 previously */ \ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index e1edd20b778d27..7273a87681b4dd 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -84,7 +84,7 @@ value = PyStackRef_DUP(value_s); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -96,7 +96,7 @@ value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -108,7 +108,7 @@ value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -120,7 +120,7 @@ value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -132,7 +132,7 @@ value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -144,7 +144,7 @@ value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -156,7 +156,7 @@ value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -168,7 +168,7 @@ value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -180,7 +180,7 @@ value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -191,7 +191,7 @@ value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -203,7 +203,7 @@ value = PyStackRef_Borrow(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -215,7 +215,7 @@ value = PyStackRef_Borrow(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -227,7 +227,7 @@ value = PyStackRef_Borrow(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -239,7 +239,7 @@ value = PyStackRef_Borrow(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -251,7 +251,7 @@ value = PyStackRef_Borrow(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -263,7 +263,7 @@ value = PyStackRef_Borrow(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -275,7 +275,7 @@ value = PyStackRef_Borrow(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -287,7 +287,7 @@ value = PyStackRef_Borrow(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -298,7 +298,7 @@ value = PyStackRef_Borrow(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -309,7 +309,7 @@ GETLOCAL(oparg) = PyStackRef_NULL; stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -320,7 +320,7 @@ value = PyStackRef_FromPyObjectBorrow(obj); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -333,7 +333,7 @@ value = PyStackRef_FromPyObjectBorrow(obj); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -346,7 +346,7 @@ value = PyStackRef_FromPyObjectBorrow(obj); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -359,7 +359,7 @@ value = PyStackRef_FromPyObjectBorrow(obj); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -372,7 +372,7 @@ value = PyStackRef_FromPyObjectBorrow(obj); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -384,7 +384,7 @@ value = PyStackRef_FromPyObjectBorrow(obj); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -396,7 +396,7 @@ _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -411,7 +411,7 @@ _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -426,7 +426,7 @@ _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -441,7 +441,7 @@ _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -456,7 +456,7 @@ _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -471,7 +471,7 @@ _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -486,7 +486,7 @@ _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -501,7 +501,7 @@ _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -515,7 +515,7 @@ _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -526,7 +526,7 @@ _PyStackRef value; value = stack_pointer[-1]; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -539,7 +539,7 @@ assert(PyStackRef_IsNull(value) || (!PyStackRef_RefcountOnObject(value)) || _Py_IsImmortal((PyStackRef_AsPyObjectBorrow(value)))); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -549,7 +549,7 @@ assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -559,7 +559,7 @@ assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -569,7 +569,7 @@ assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -579,12 +579,12 @@ tos = stack_pointer[-1]; nos = stack_pointer[-2]; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(tos); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(nos); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -596,7 +596,7 @@ res = PyStackRef_NULL; stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -604,7 +604,7 @@ _PyStackRef value; value = stack_pointer[-1]; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -618,7 +618,7 @@ iter = stack_pointer[-2]; (void)index_or_null; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iter); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -634,7 +634,7 @@ val = value; stack_pointer[-2] = val; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(receiver); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -649,7 +649,7 @@ PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -659,7 +659,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -682,7 +682,7 @@ int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -692,7 +692,7 @@ res = err ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -723,7 +723,7 @@ } else { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -833,7 +833,7 @@ else { assert(Py_SIZE(value_o)); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -849,14 +849,14 @@ _PyStackRef res; value = stack_pointer[-1]; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_True; stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -868,7 +868,7 @@ PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -878,7 +878,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -949,7 +949,7 @@ PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -974,7 +974,7 @@ PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -999,7 +999,7 @@ PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1043,12 +1043,12 @@ if (PyStackRef_IsNull(res)) { stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1070,12 +1070,12 @@ if (PyStackRef_IsNull(res)) { stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1097,12 +1097,12 @@ if (PyStackRef_IsNull(res)) { stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1124,12 +1124,12 @@ if (PyStackRef_IsNull(res)) { stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1151,12 +1151,12 @@ if (PyStackRef_IsNull(res)) { stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1178,12 +1178,12 @@ if (PyStackRef_IsNull(res)) { stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1203,13 +1203,13 @@ PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); if (res_o == NULL) { stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1240,7 +1240,7 @@ PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); PyObject *right_o = PyStackRef_AsPyObjectSteal(right); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyUnicode_Append(&temp, right_o); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1304,11 +1304,11 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1330,7 +1330,7 @@ } else { stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); Py_DECREF(slice); @@ -1338,7 +1338,7 @@ stack_pointer += 2; } stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(container); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1348,7 +1348,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1371,7 +1371,7 @@ } else { stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); Py_DECREF(slice); @@ -1389,7 +1389,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -4; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { JUMP_TO_ERROR(); } @@ -1443,7 +1443,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1472,14 +1472,14 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1511,14 +1511,14 @@ PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(str_st); stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_FromPyObjectBorrow(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1569,7 +1569,7 @@ PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); res = PyStackRef_FromPyObjectNew(res_o); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp = tuple_st; tuple_st = res; @@ -1631,14 +1631,14 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (rc <= 0) { JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1673,7 +1673,7 @@ STAT_INC(BINARY_OP, hit); stack_pointer[0] = getitem; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1692,7 +1692,7 @@ new_frame = PyStackRef_Wrap(pushed_frame); stack_pointer[-3] = new_frame; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1706,11 +1706,11 @@ PyStackRef_AsPyObjectSteal(v)); if (err < 0) { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1726,11 +1726,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1757,7 +1757,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { JUMP_TO_ERROR(); } @@ -1799,7 +1799,7 @@ UNLOCK_OBJECT(list); PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(list_st); Py_DECREF(old_value); @@ -1823,7 +1823,7 @@ PyStackRef_AsPyObjectSteal(value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(dict_st); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1851,7 +1851,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { JUMP_TO_ERROR(); } @@ -1868,7 +1868,7 @@ PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1878,7 +1878,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1904,14 +1904,14 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1922,7 +1922,7 @@ assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); assert(STACK_LEVEL() == 0); _Py_LeaveRecursiveCallPy(tstate); @@ -1935,7 +1935,7 @@ LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1958,7 +1958,7 @@ type->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(obj); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1968,7 +1968,7 @@ iter_o = (*getter)(obj_o); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(obj); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1989,7 +1989,7 @@ iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[0] = iter; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2006,7 +2006,7 @@ awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); stack_pointer[0] = awaitable; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2019,7 +2019,7 @@ PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2029,7 +2029,7 @@ iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[0] = iter; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2078,7 +2078,7 @@ gen->gi_frame_state = FRAME_SUSPENDED + oparg; _PyStackRef temp = retval; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; @@ -2101,7 +2101,7 @@ LLTRACE_RESUME_FRAME(); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2115,7 +2115,7 @@ ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2126,7 +2126,7 @@ value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2149,7 +2149,7 @@ bc = PyStackRef_FromPyObjectSteal(bc_o); stack_pointer[0] = bc; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2166,7 +2166,7 @@ "no locals found when storing %R", name); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2183,7 +2183,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2227,7 +2227,7 @@ top = &stack_pointer[-1 + oparg]; PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg, -1, top); Py_DECREF(seq_o); @@ -2236,7 +2236,7 @@ JUMP_TO_ERROR(); } stack_pointer += oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2259,7 +2259,7 @@ stack_pointer[-1] = val1; stack_pointer[0] = val0; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(seq); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2284,7 +2284,7 @@ *values++ = PyStackRef_FromPyObjectNew(items[i]); } stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(seq); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2317,7 +2317,7 @@ } UNLOCK_OBJECT(seq_o); stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(seq); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2332,7 +2332,7 @@ top = &stack_pointer[(oparg & 0xFF) + (oparg >> 8)]; PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg & 0xFF, oparg >> 8, top); Py_DECREF(seq_o); @@ -2341,7 +2341,7 @@ JUMP_TO_ERROR(); } stack_pointer += 1 + (oparg & 0xFF) + (oparg >> 8); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2365,7 +2365,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { JUMP_TO_ERROR(); } @@ -2381,7 +2381,7 @@ int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2400,7 +2400,7 @@ int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2442,7 +2442,7 @@ locals = PyStackRef_FromPyObjectNew(l); stack_pointer[0] = locals; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2461,7 +2461,7 @@ v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[0] = v; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2477,7 +2477,7 @@ JUMP_TO_ERROR(); } stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2489,7 +2489,7 @@ null[0] = PyStackRef_NULL; } stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2543,7 +2543,7 @@ STAT_INC(LOAD_GLOBAL, hit); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2580,7 +2580,7 @@ STAT_INC(LOAD_GLOBAL, hit); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2663,14 +2663,14 @@ } } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(class_dict_st); stack_pointer = _PyFrame_GetStackPointer(frame); value = PyStackRef_FromPyObjectSteal(value_o); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2684,7 +2684,7 @@ if (PyStackRef_IsNull(value)) { stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2692,7 +2692,7 @@ } stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2705,7 +2705,7 @@ PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2729,39 +2729,18 @@ _PyStackRef str; oparg = CURRENT_OPARG(); pieces = &stack_pointer[-oparg]; - STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); - if (CONVERSION_FAILED(pieces_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = pieces[_i]; - pieces[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } - PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); - STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = pieces[_i]; - pieces[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } + PyObject *str_o = _Py_BuildString_StackRefSteal(pieces, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); if (str_o == NULL) { + stack_pointer += -oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } str = PyStackRef_FromPyObjectSteal(str_o); - stack_pointer[0] = str; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-oparg] = str; + stack_pointer += 1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2789,7 +2768,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (oparg & 1) { stack_pointer += -(oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(format[0]); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2798,12 +2777,12 @@ stack_pointer += -(oparg & 1); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(str); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2813,7 +2792,7 @@ interpolation = PyStackRef_FromPyObjectSteal(interpolation_o); stack_pointer[0] = interpolation; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2829,12 +2808,12 @@ PyObject *template_o = _PyTemplate_Build(strings_o, interpolations_o); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(interpolations); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(strings); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2844,7 +2823,7 @@ template = PyStackRef_FromPyObjectSteal(template_o); stack_pointer[0] = template; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2860,7 +2839,7 @@ tup = PyStackRef_FromPyObjectStealMortal(tup_o); stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2878,7 +2857,7 @@ list = PyStackRef_FromPyObjectStealMortal(list_o); stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2908,7 +2887,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable_st); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2916,7 +2895,7 @@ } assert(Py_IsNone(none_val)); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable_st); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2934,7 +2913,7 @@ PyStackRef_AsPyObjectBorrow(iterable)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2962,7 +2941,7 @@ } stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } int err = 0; @@ -2982,7 +2961,7 @@ } if (err) { stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(set_o); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2991,7 +2970,7 @@ set = PyStackRef_FromPyObjectStealMortal(set_o); stack_pointer[-oparg] = set; stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3000,44 +2979,18 @@ _PyStackRef map; oparg = CURRENT_OPARG(); values = &stack_pointer[-oparg*2]; - STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); - if (CONVERSION_FAILED(values_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg*2; --_i >= 0;) { - tmp = values[_i]; - values[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg*2; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *map_o = _PyDict_FromItems( - values_o, 2, - values_o+1, 2, - oparg); + PyObject *map_o = _Py_BuildMap_StackRefSteal(values, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg*2; --_i >= 0;) { - tmp = values[_i]; - values[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg*2; - assert(WITHIN_STACK_BOUNDS()); if (map_o == NULL) { + stack_pointer += -oparg*2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } map = PyStackRef_FromPyObjectStealMortal(map_o); - stack_pointer[0] = map; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-oparg*2] = map; + stack_pointer += 1 - oparg*2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3103,14 +3056,14 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(update); stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_ERROR(); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(update); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -3136,14 +3089,14 @@ _PyEval_FormatKwargsError(tstate, callable_o, update_o); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(update); stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_ERROR(); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(update); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -3169,11 +3122,11 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3216,14 +3169,14 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (attr == NULL) { JUMP_TO_ERROR(); } attr_st = PyStackRef_FromPyObjectSteal(attr); stack_pointer[0] = attr_st; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3264,7 +3217,7 @@ self_or_null = self_st; } else { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(self_st); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -3272,7 +3225,7 @@ stack_pointer += 1; } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp = global_super_st; global_super_st = self_or_null; @@ -3284,12 +3237,12 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); attr = PyStackRef_FromPyObjectSteal(attr_o); stack_pointer[0] = attr; stack_pointer[1] = self_or_null; stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3313,7 +3266,7 @@ } else { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -3329,7 +3282,7 @@ PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -3340,7 +3293,7 @@ stack_pointer += 1; } stack_pointer += (oparg&1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3672,7 +3625,7 @@ } UNLOCK_OBJECT(owner_o); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); Py_XDECREF(old_value); @@ -3731,7 +3684,7 @@ UNLOCK_OBJECT(dict); STAT_INC(STORE_ATTR, hit); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); Py_XDECREF(old_value); @@ -3756,7 +3709,7 @@ FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); UNLOCK_OBJECT(owner_o); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); Py_XDECREF(old_value); @@ -3786,7 +3739,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_ERROR(); } @@ -3805,7 +3758,7 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3827,7 +3780,7 @@ res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3853,7 +3806,7 @@ res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3877,7 +3830,7 @@ res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3900,11 +3853,11 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); b = res ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = b; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3929,14 +3882,14 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res < 0) { JUMP_TO_ERROR(); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = b; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3974,14 +3927,14 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res < 0) { JUMP_TO_ERROR(); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = b; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4008,14 +3961,14 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res < 0) { JUMP_TO_ERROR(); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = b; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4043,7 +3996,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } PyObject *match_o = NULL; @@ -4061,7 +4014,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res < 0) { JUMP_TO_ERROR(); } @@ -4079,7 +4032,7 @@ stack_pointer[0] = rest; stack_pointer[1] = match; stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4102,14 +4055,14 @@ int res = PyErr_GivenExceptionMatches(left_o, right_o); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(right); stack_pointer = _PyFrame_GetStackPointer(frame); b = res ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = b; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4135,14 +4088,14 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4161,7 +4114,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4208,7 +4161,7 @@ len = PyStackRef_FromPyObjectSteal(len_o); stack_pointer[0] = len; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4241,7 +4194,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (attrs_o) { assert(PyTuple_CheckExact(attrs_o)); attrs = PyStackRef_FromPyObjectSteal(attrs_o); @@ -4254,7 +4207,7 @@ } stack_pointer[0] = attrs; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4266,7 +4219,7 @@ res = match ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4278,7 +4231,7 @@ res = match ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4298,7 +4251,7 @@ values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); stack_pointer[0] = values_or_none; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4323,7 +4276,7 @@ PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -4337,7 +4290,7 @@ stack_pointer[-1] = iter; stack_pointer[0] = index_or_null; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4403,7 +4356,7 @@ stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4476,7 +4429,7 @@ stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4525,7 +4478,7 @@ stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4580,7 +4533,7 @@ next = PyStackRef_FromPyObjectSteal(res); stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4616,7 +4569,7 @@ gen_frame = PyStackRef_Wrap(pushed_frame); stack_pointer[0] = gen_frame; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4628,7 +4581,7 @@ method_and_self[1] = self; method_and_self[0] = PyStackRef_NULL; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4693,7 +4646,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4715,7 +4668,7 @@ stack_pointer[-1] = prev_exc; stack_pointer[0] = new_exc; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4762,7 +4715,7 @@ stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4783,7 +4736,7 @@ stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4797,14 +4750,14 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); attr = PyStackRef_FromPyObjectNew(descr); stack_pointer[0] = attr; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4819,14 +4772,14 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); attr = PyStackRef_FromPyObjectNew(descr); stack_pointer[0] = attr; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4859,7 +4812,7 @@ stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -4916,14 +4869,14 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { JUMP_TO_ERROR(); } new_frame = PyStackRef_Wrap(temp); stack_pointer[0] = new_frame; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5034,68 +4987,28 @@ #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); + PyObject *res_o = _Py_VectorCall_StackRefSteal( + callable, + arguments, + total_args, + PyStackRef_NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5204,7 +5117,7 @@ new_frame = PyStackRef_Wrap(pushed_frame); stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5229,7 +5142,7 @@ new_frame = PyStackRef_Wrap(pushed_frame); stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5254,7 +5167,7 @@ new_frame = PyStackRef_Wrap(pushed_frame); stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5279,7 +5192,7 @@ new_frame = PyStackRef_Wrap(pushed_frame); stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5304,7 +5217,7 @@ new_frame = PyStackRef_Wrap(pushed_frame); stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5328,7 +5241,7 @@ new_frame = PyStackRef_Wrap(pushed_frame); stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5338,7 +5251,7 @@ assert(tstate->interp->eval_frame == NULL); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); @@ -5409,7 +5322,7 @@ res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); stack_pointer[-3] = res; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5445,7 +5358,7 @@ (void)callable; (void)null; stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5455,7 +5368,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5488,7 +5401,7 @@ (void)callable; (void)null; stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5498,7 +5411,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5572,7 +5485,7 @@ tstate, init, NULL, args-1, oparg+1, NULL, shim); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FrameClearAndPop(tstate, shim); @@ -5584,7 +5497,7 @@ init_frame = PyStackRef_Wrap(temp); stack_pointer[0] = init_frame; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5600,7 +5513,7 @@ JUMP_TO_ERROR(); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5630,57 +5543,21 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); + PyObject *res_o = _Py_CallBuiltinClass_StackRefSteal( + callable, + arguments, + total_args); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5727,7 +5604,7 @@ PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5737,7 +5614,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5750,13 +5627,13 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -5766,62 +5643,22 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunctionFast_CAST(cfunc)( - PyCFunction_GET_SELF(callable_o), - args_o, - total_args); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); + PyObject *res_o = _Py_BuiltinCallFast_StackRefSteal( + callable, + arguments, + total_args + ); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5834,13 +5671,13 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -5851,61 +5688,17 @@ } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFastWithKeywords cfunc = - _PyCFunctionFastWithKeywords_CAST(PyCFunction_GET_FUNCTION(callable_o)); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); + PyObject *res_o = _Py_BuiltinCallFastWithKeywords_StackRefSteal(callable, arguments, total_args); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5944,19 +5737,19 @@ JUMP_TO_ERROR(); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -5993,17 +5786,17 @@ } (void)null; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(cls); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(instance); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6011,7 +5804,7 @@ assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6045,12 +5838,12 @@ int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); UNLOCK_OBJECT(self_o); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(self); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6132,14 +5925,14 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6181,61 +5974,24 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); - int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFastWithKeywords cfunc = - _PyCFunctionFastWithKeywords_CAST(meth->ml_meth); - PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); + PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRefSteal( + callable, + meth, + self, + arguments, + total_args + ); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6290,7 +6046,7 @@ PyStackRef_CLOSE(self_stackref); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6300,7 +6056,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6341,60 +6097,24 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(CALL, hit); - int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFast cfunc = _PyCFunctionFast_CAST(meth->ml_meth); - PyObject *res_o = cfunc(self, (args_o + 1), nargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); + PyObject *res_o = _PyCallMethodDescriptorFast_StackRefSteal( + callable, + meth, + self, + arguments, + total_args + ); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6456,19 +6176,19 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(kwnames); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { JUMP_TO_ERROR(); } new_frame = PyStackRef_Wrap(temp); stack_pointer[0] = new_frame; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6569,78 +6289,28 @@ #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = kwnames; - kwnames = PyStackRef_NULL; - stack_pointer[-1] = kwnames; - PyStackRef_CLOSE(tmp); - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-2 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-3 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_ERROR(); - } - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(kwnames); + PyObject *res_o = _Py_VectorCall_StackRefSteal( + callable, + arguments, + total_args, + kwnames); stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -3 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6686,7 +6356,7 @@ PyFunction_New(codeobj, GLOBALS()); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(codeobj_st); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6698,7 +6368,7 @@ func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); stack_pointer[0] = func; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6720,7 +6390,7 @@ *ptr = attr; stack_pointer[-2] = func_out; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6752,7 +6422,7 @@ LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6774,14 +6444,14 @@ } stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (slice_o == NULL) { JUMP_TO_ERROR(); } slice = PyStackRef_FromPyObjectStealMortal(slice_o); stack_pointer[0] = slice; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6797,7 +6467,7 @@ PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6807,7 +6477,7 @@ result = PyStackRef_FromPyObjectSteal(result_o); stack_pointer[0] = result; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6821,7 +6491,7 @@ PyObject *res_o = PyObject_Format(value_o, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6836,7 +6506,7 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6858,14 +6528,14 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6876,7 +6546,7 @@ top = PyStackRef_DUP(bottom); stack_pointer[0] = top; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6887,7 +6557,7 @@ top = PyStackRef_DUP(bottom); stack_pointer[0] = top; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6898,7 +6568,7 @@ top = PyStackRef_DUP(bottom); stack_pointer[0] = top; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6910,7 +6580,7 @@ top = PyStackRef_DUP(bottom); stack_pointer[0] = top; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -6942,7 +6612,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -7009,7 +6679,7 @@ flag = stack_pointer[-1]; int is_true = PyStackRef_IsTrue(flag); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (!is_true) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -7022,7 +6692,7 @@ flag = stack_pointer[-1]; int is_false = PyStackRef_IsFalse(flag); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (!is_false) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -7036,7 +6706,7 @@ int is_none = PyStackRef_IsNone(val); if (!is_none) { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(val); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -7046,7 +6716,7 @@ } } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -7055,7 +6725,7 @@ val = stack_pointer[-1]; int is_none = PyStackRef_IsNone(val); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(val); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -7162,7 +6832,7 @@ value = PyStackRef_FromPyObjectNew(ptr); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -7172,14 +6842,14 @@ pop = stack_pointer[-1]; PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop); stack_pointer = _PyFrame_GetStackPointer(frame); value = PyStackRef_FromPyObjectNew(ptr); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -7189,7 +6859,7 @@ value = PyStackRef_FromPyObjectBorrow(ptr); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -7200,7 +6870,7 @@ callable = stack_pointer[-2]; (void)null; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -7215,13 +6885,13 @@ null = stack_pointer[-2]; callable = stack_pointer[-3]; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop); stack_pointer = _PyFrame_GetStackPointer(frame); (void)null; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -7238,18 +6908,18 @@ null = stack_pointer[-3]; callable = stack_pointer[-4]; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop2); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop1); stack_pointer = _PyFrame_GetStackPointer(frame); (void)null; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -7262,14 +6932,14 @@ pop = stack_pointer[-1]; PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop); stack_pointer = _PyFrame_GetStackPointer(frame); value = PyStackRef_FromPyObjectBorrow(ptr); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -7281,19 +6951,19 @@ pop1 = stack_pointer[-2]; PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop2); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop1); stack_pointer = _PyFrame_GetStackPointer(frame); value = PyStackRef_FromPyObjectBorrow(ptr); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -7306,14 +6976,14 @@ PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); (void)null; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); value = PyStackRef_FromPyObjectBorrow(ptr); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -7327,20 +6997,20 @@ callable = stack_pointer[-3]; PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop); stack_pointer = _PyFrame_GetStackPointer(frame); (void)null; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); value = PyStackRef_FromPyObjectBorrow(ptr); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -7356,25 +7026,25 @@ callable = stack_pointer[-4]; PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop2); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop1); stack_pointer = _PyFrame_GetStackPointer(frame); (void)null; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); value = PyStackRef_FromPyObjectBorrow(ptr); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -7389,7 +7059,7 @@ stack_pointer[-1] = value; stack_pointer[0] = new; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -7404,7 +7074,7 @@ stack_pointer[-1] = value; stack_pointer[0] = new; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 57d5e71144d38c..68d73cccec4d6b 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -76,7 +76,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); } DISPATCH(); } @@ -135,7 +135,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -195,7 +195,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -255,7 +255,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -314,12 +314,12 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -386,7 +386,7 @@ PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); PyObject *right_o = PyStackRef_AsPyObjectSteal(right); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyUnicode_Append(&temp, right_o); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -460,7 +460,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -520,7 +520,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -578,7 +578,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (rc <= 0) { JUMP_TO_LABEL(error); } @@ -586,7 +586,7 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -661,7 +661,7 @@ assert(tstate->interp->eval_frame == NULL); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); @@ -759,7 +759,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); } DISPATCH(); } @@ -824,7 +824,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -832,7 +832,7 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -902,7 +902,7 @@ PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(str_st); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -910,7 +910,7 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -976,7 +976,7 @@ PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); res = PyStackRef_FromPyObjectNew(res_o); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp = tuple_st; tuple_st = res; @@ -1041,7 +1041,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -1101,7 +1101,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -1138,7 +1138,7 @@ } else { stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); Py_DECREF(slice); @@ -1146,7 +1146,7 @@ stack_pointer += 2; } stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(container); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1157,7 +1157,7 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -1191,7 +1191,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (oparg & 1) { stack_pointer += -(oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(format[0]); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1200,12 +1200,12 @@ stack_pointer += -(oparg & 1); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(str); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1215,7 +1215,7 @@ interpolation = PyStackRef_FromPyObjectSteal(interpolation_o); stack_pointer[0] = interpolation; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -1239,7 +1239,7 @@ list = PyStackRef_FromPyObjectStealMortal(list_o); stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -1254,44 +1254,18 @@ _PyStackRef *values; _PyStackRef map; values = &stack_pointer[-oparg*2]; - STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); - if (CONVERSION_FAILED(values_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg*2; --_i >= 0;) { - tmp = values[_i]; - values[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg*2; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *map_o = _PyDict_FromItems( - values_o, 2, - values_o+1, 2, - oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg*2; --_i >= 0;) { - tmp = values[_i]; - values[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } + PyObject *map_o = _Py_BuildMap_StackRefSteal(values, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg*2; - assert(WITHIN_STACK_BOUNDS()); if (map_o == NULL) { + stack_pointer += -oparg*2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } map = PyStackRef_FromPyObjectStealMortal(map_o); - stack_pointer[0] = map; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-oparg*2] = map; + stack_pointer += 1 - oparg*2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -1319,7 +1293,7 @@ } stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } int err = 0; @@ -1339,7 +1313,7 @@ } if (err) { stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(set_o); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1348,7 +1322,7 @@ set = PyStackRef_FromPyObjectStealMortal(set_o); stack_pointer[-oparg] = set; stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -1376,14 +1350,14 @@ } stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (slice_o == NULL) { JUMP_TO_LABEL(error); } slice = PyStackRef_FromPyObjectStealMortal(slice_o); stack_pointer[0] = slice; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -1398,39 +1372,18 @@ _PyStackRef *pieces; _PyStackRef str; pieces = &stack_pointer[-oparg]; - STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); - if (CONVERSION_FAILED(pieces_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = pieces[_i]; - pieces[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } - PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); - STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = pieces[_i]; - pieces[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } + PyObject *str_o = _Py_BuildString_StackRefSteal(pieces, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); if (str_o == NULL) { + stack_pointer += -oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } str = PyStackRef_FromPyObjectSteal(str_o); - stack_pointer[0] = str; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-oparg] = str; + stack_pointer += 1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -1453,12 +1406,12 @@ PyObject *template_o = _PyTemplate_Build(strings_o, interpolations_o); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(interpolations); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(strings); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1468,7 +1421,7 @@ template = PyStackRef_FromPyObjectSteal(template_o); stack_pointer[0] = template; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -1490,7 +1443,7 @@ tup = PyStackRef_FromPyObjectStealMortal(tup_o); stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -1583,7 +1536,7 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { JUMP_TO_LABEL(error); } @@ -1611,7 +1564,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } stack_pointer[-2 - oparg] = callable; @@ -1664,7 +1617,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -1674,7 +1627,7 @@ { stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -1778,7 +1731,7 @@ tstate, init, NULL, args-1, oparg+1, NULL, shim); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FrameClearAndPop(tstate, shim); @@ -1936,7 +1889,7 @@ assert(tstate->interp->eval_frame == NULL); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); @@ -2046,7 +1999,7 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { JUMP_TO_LABEL(error); } @@ -2118,60 +2071,24 @@ JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); + PyObject *res_o = _Py_CallBuiltinClass_StackRefSteal( + callable, + arguments, + total_args); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } // _CHECK_PERIODIC_AT_END { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2204,13 +2121,13 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); @@ -2222,65 +2139,25 @@ JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunctionFast_CAST(cfunc)( - PyCFunction_GET_SELF(callable_o), - args_o, - total_args); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); + PyObject *res_o = _Py_BuiltinCallFast_StackRefSteal( + callable, + arguments, + total_args + ); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } // _CHECK_PERIODIC_AT_END { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2313,13 +2190,13 @@ args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); @@ -2332,64 +2209,20 @@ } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFastWithKeywords cfunc = - _PyCFunctionFastWithKeywords_CAST(PyCFunction_GET_FUNCTION(callable_o)); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); + PyObject *res_o = _Py_BuiltinCallFastWithKeywords_StackRefSteal(callable, arguments, total_args); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } // _CHECK_PERIODIC_AT_END { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2460,7 +2293,7 @@ PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2473,7 +2306,7 @@ { stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2592,14 +2425,14 @@ int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( tstate, func_st, locals, nargs, callargs, kwargs, frame); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { JUMP_TO_LABEL(error); } @@ -2617,17 +2450,17 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(kwargs_st); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callargs_st); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(func_st); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2640,7 +2473,7 @@ { stack_pointer[0] = result; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2667,7 +2500,7 @@ PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2677,7 +2510,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -2709,14 +2542,14 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -2773,17 +2606,17 @@ } (void)null; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(cls); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(instance); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2792,7 +2625,7 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -2876,7 +2709,7 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(kwnames); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -2911,7 +2744,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } stack_pointer[-3 - oparg] = callable; @@ -2966,7 +2799,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -2974,7 +2807,7 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -3072,12 +2905,12 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(kwnames); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { JUMP_TO_LABEL(error); } @@ -3150,81 +2983,31 @@ #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = kwnames; - kwnames = PyStackRef_NULL; - stack_pointer[-1] = kwnames; - PyStackRef_CLOSE(tmp); - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-2 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-3 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); + PyObject *res_o = _Py_VectorCall_StackRefSteal( + callable, + arguments, + total_args, + kwnames); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -3 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } // _CHECK_PERIODIC_AT_END { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-3 - oparg] = res; + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -3309,12 +3092,12 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(kwnames); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { JUMP_TO_LABEL(error); } @@ -3400,12 +3183,12 @@ JUMP_TO_LABEL(error); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -3413,7 +3196,7 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -3479,12 +3262,12 @@ int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); UNLOCK_OBJECT(self_o); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(self); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -3549,68 +3332,32 @@ PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); assert(self != NULL); if (!Py_IS_TYPE(self, method->d_common.d_type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); - } - STAT_INC(CALL, hit); - int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); } + STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFast cfunc = _PyCFunctionFast_CAST(meth->ml_meth); - PyObject *res_o = cfunc(self, (args_o + 1), nargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); + PyObject *res_o = _PyCallMethodDescriptorFast_StackRefSteal( + callable, + meth, + self, + arguments, + total_args + ); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } // _CHECK_PERIODIC_AT_END { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -3676,64 +3423,27 @@ JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); - int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFastWithKeywords cfunc = - _PyCFunctionFastWithKeywords_CAST(meth->ml_meth); - PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); + PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRefSteal( + callable, + meth, + self, + arguments, + total_args + ); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } // _CHECK_PERIODIC_AT_END { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -3813,7 +3523,7 @@ PyStackRef_CLOSE(self_stackref); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -3826,7 +3536,7 @@ { stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -3922,7 +3632,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -3932,7 +3642,7 @@ { stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -3983,71 +3693,31 @@ #if TIER_ONE assert(opcode != INSTRUMENTED_CALL); #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; _PyStackRef *arguments = args; if (!PyStackRef_IsNull(self_or_null)) { arguments--; total_args++; } - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); + PyObject *res_o = _Py_VectorCall_StackRefSteal( + callable, + arguments, + total_args, + PyStackRef_NULL); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { + stack_pointer += -2 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } // _CHECK_PERIODIC_AT_END { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -4158,7 +3828,7 @@ assert(tstate->interp->eval_frame == NULL); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); @@ -4240,7 +3910,7 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { JUMP_TO_LABEL(error); } @@ -4319,7 +3989,7 @@ (void)callable; (void)null; stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -4332,7 +4002,7 @@ { stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -4391,7 +4061,7 @@ (void)callable; (void)null; stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -4404,7 +4074,7 @@ { stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -4462,7 +4132,7 @@ res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); stack_pointer[-3] = res; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -4501,7 +4171,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } PyObject *match_o = NULL; @@ -4519,7 +4189,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res < 0) { JUMP_TO_LABEL(error); } @@ -4537,7 +4207,7 @@ stack_pointer[0] = rest; stack_pointer[1] = match; stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -4567,14 +4237,14 @@ int res = PyErr_GivenExceptionMatches(left_o, right_o); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(right); stack_pointer = _PyFrame_GetStackPointer(frame); b = res ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = b; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -4621,7 +4291,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); none = PyStackRef_None; } else { @@ -4633,7 +4303,7 @@ stack_pointer[0] = none; stack_pointer[1] = value; stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -4686,7 +4356,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -4706,7 +4376,7 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -4761,7 +4431,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -4820,7 +4490,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -4879,7 +4549,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -4931,7 +4601,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res < 0) { JUMP_TO_LABEL(error); } @@ -4939,7 +4609,7 @@ } stack_pointer[0] = b; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -4989,7 +4659,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res < 0) { JUMP_TO_LABEL(error); } @@ -4997,7 +4667,7 @@ } stack_pointer[0] = b; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -5047,7 +4717,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res < 0) { JUMP_TO_LABEL(error); } @@ -5055,7 +4725,7 @@ } stack_pointer[0] = b; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -5077,7 +4747,7 @@ PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5087,7 +4757,7 @@ result = PyStackRef_FromPyObjectSteal(result_o); stack_pointer[0] = result; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -5105,7 +4775,7 @@ top = PyStackRef_DUP(bottom); stack_pointer[0] = top; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -5145,7 +4815,7 @@ int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5285,7 +4955,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { JUMP_TO_LABEL(error); } @@ -5317,14 +4987,14 @@ _PyEval_FormatKwargsError(tstate, callable_o, update_o); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(update); stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_LABEL(error); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(update); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5360,14 +5030,14 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(update); stack_pointer = _PyFrame_GetStackPointer(frame); JUMP_TO_LABEL(error); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(update); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5407,7 +5077,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); } else { Py_INCREF(exc); @@ -5429,7 +5099,7 @@ _PyStackRef value; value = stack_pointer[-1]; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5452,7 +5122,7 @@ val = value; stack_pointer[-2] = val; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(receiver); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5518,7 +5188,7 @@ JUMP_TO_LABEL(error); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -5555,7 +5225,7 @@ PyObject *res_o = PyObject_Format(value_o, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5570,7 +5240,7 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -5599,14 +5269,14 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -5660,7 +5330,7 @@ stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -5815,7 +5485,7 @@ stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -5882,7 +5552,7 @@ } stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -5938,7 +5608,7 @@ stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -5968,7 +5638,7 @@ type->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(obj); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5978,7 +5648,7 @@ iter_o = (*getter)(obj_o); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(obj); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -5999,7 +5669,7 @@ iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[0] = iter; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -6023,7 +5693,7 @@ awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); stack_pointer[0] = awaitable; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -6042,7 +5712,7 @@ PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6052,7 +5722,7 @@ iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[0] = iter; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -6084,7 +5754,7 @@ PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6098,7 +5768,7 @@ stack_pointer[-1] = iter; stack_pointer[0] = index_or_null; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -6126,7 +5796,7 @@ len = PyStackRef_FromPyObjectSteal(len_o); stack_pointer[0] = len; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -6196,7 +5866,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -6228,14 +5898,14 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -6329,7 +5999,7 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { JUMP_TO_LABEL(error); } @@ -6355,7 +6025,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -6406,7 +6076,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -6416,7 +6086,7 @@ { stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6535,14 +6205,14 @@ int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( tstate, func_st, locals, nargs, callargs, kwargs, frame); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { JUMP_TO_LABEL(error); } @@ -6560,17 +6230,17 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(kwargs_st); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callargs_st); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(func_st); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6583,7 +6253,7 @@ { stack_pointer[0] = result; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6682,7 +6352,7 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(kwnames); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6715,7 +6385,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -6768,7 +6438,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -6776,7 +6446,7 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -6820,7 +6490,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); } else { Py_INCREF(exc); @@ -6855,7 +6525,7 @@ } } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6889,7 +6559,7 @@ val = value; stack_pointer[-2] = val; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(receiver); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -6928,7 +6598,7 @@ stack_pointer[-1] = null_or_index; stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -7088,7 +6758,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } } @@ -7133,7 +6803,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (super == NULL) { JUMP_TO_LABEL(error); } @@ -7156,7 +6826,7 @@ } stack_pointer[0] = attr; stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -7194,7 +6864,7 @@ (void)index_or_null; INSTRUMENTED_JUMP(prev_instr, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iter); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -7221,7 +6891,7 @@ INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -7245,14 +6915,14 @@ } else { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += 1; } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -7273,7 +6943,7 @@ RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); if (jump) { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -7305,7 +6975,7 @@ INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -7420,7 +7090,7 @@ assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); assert(STACK_LEVEL() == 0); _Py_LeaveRecursiveCallPy(tstate); @@ -7434,7 +7104,7 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -7478,7 +7148,7 @@ gen->gi_frame_state = FRAME_SUSPENDED + oparg; _PyStackRef temp = retval; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; @@ -7502,7 +7172,7 @@ } stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -7530,7 +7200,7 @@ if (!PyStackRef_IsNull(executor)) { tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(executor); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -7566,11 +7236,11 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); b = res ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = b; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -7738,7 +7408,7 @@ JUMP_TO_LABEL(pop_1_error); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -7774,7 +7444,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable_st); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -7782,7 +7452,7 @@ } assert(Py_IsNone(none_val)); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable_st); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -7838,7 +7508,7 @@ } else { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -7854,7 +7524,7 @@ PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -7866,7 +7536,7 @@ } } stack_pointer += (oparg&1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -7924,7 +7594,7 @@ } } stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -7992,7 +7662,7 @@ } } stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8048,7 +7718,7 @@ tstate, PyStackRef_FromPyObjectNew(f), 2, frame); new_frame->localsplus[0] = owner; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); new_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name); frame->return_offset = 10u ; DISPATCH_INLINED(new_frame); @@ -8130,7 +7800,7 @@ } } stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8186,7 +7856,7 @@ stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8232,7 +7902,7 @@ stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8299,7 +7969,7 @@ stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8373,7 +8043,7 @@ } } stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8412,7 +8082,7 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -8420,7 +8090,7 @@ } stack_pointer[0] = attr; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8480,7 +8150,7 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -8488,7 +8158,7 @@ } stack_pointer[0] = attr; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8573,7 +8243,7 @@ assert(tstate->interp->eval_frame == NULL); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); @@ -8651,7 +8321,7 @@ } } stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8761,7 +8431,7 @@ } } stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8791,7 +8461,7 @@ bc = PyStackRef_FromPyObjectSteal(bc_o); stack_pointer[0] = bc; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8808,7 +8478,7 @@ value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8825,7 +8495,7 @@ value = PyStackRef_FromPyObjectBorrow(obj); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8845,7 +8515,7 @@ if (PyStackRef_IsNull(value)) { stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -8853,7 +8523,7 @@ } stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8870,7 +8540,7 @@ value = PyStackRef_DUP(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8887,7 +8557,7 @@ GETLOCAL(oparg) = PyStackRef_NULL; stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8904,7 +8574,7 @@ value = PyStackRef_Borrow(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8925,7 +8595,7 @@ stack_pointer[0] = value1; stack_pointer[1] = value2; stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8951,7 +8621,7 @@ value = PyStackRef_DUP(value_s); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -8972,7 +8642,7 @@ stack_pointer[0] = value1; stack_pointer[1] = value2; stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -9010,14 +8680,14 @@ } } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(class_dict_st); stack_pointer = _PyFrame_GetStackPointer(frame); value = PyStackRef_FromPyObjectSteal(value_o); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -9038,7 +8708,7 @@ int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(mod_or_class_dict); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -9092,7 +8762,7 @@ v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[0] = v; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -9148,7 +8818,7 @@ } } stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -9228,7 +8898,7 @@ } stack_pointer[0] = res; stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -9295,7 +8965,7 @@ } stack_pointer[0] = res; stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -9319,7 +8989,7 @@ locals = PyStackRef_FromPyObjectNew(l); stack_pointer[0] = locals; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -9342,7 +9012,7 @@ v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[0] = v; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -9360,7 +9030,7 @@ value = PyStackRef_FromPyObjectBorrow(obj); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -9386,7 +9056,7 @@ method_and_self = &stack_pointer[-1]; PyObject *name = _Py_SpecialMethods[oparg].name; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int err = _PyObject_LookupSpecialMethod(name, method_and_self); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -9475,7 +9145,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } } @@ -9520,7 +9190,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (super == NULL) { JUMP_TO_LABEL(error); } @@ -9543,7 +9213,7 @@ } stack_pointer[0] = attr; stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -9598,14 +9268,14 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (attr == NULL) { JUMP_TO_LABEL(error); } attr_st = PyStackRef_FromPyObjectSteal(attr); stack_pointer[0] = attr_st; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -9658,7 +9328,7 @@ self_or_null = self_st; } else { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(self_st); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -9666,7 +9336,7 @@ stack_pointer += 1; } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef tmp = global_super_st; global_super_st = self_or_null; @@ -9678,12 +9348,12 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); attr = PyStackRef_FromPyObjectSteal(attr_o); stack_pointer[0] = attr; stack_pointer[1] = self_or_null; stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -9725,7 +9395,7 @@ PyFunction_New(codeobj, GLOBALS()); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(codeobj_st); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -9737,7 +9407,7 @@ func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); stack_pointer[0] = func; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -9768,7 +9438,7 @@ JUMP_TO_LABEL(pop_2_error); } stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -9807,7 +9477,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (attrs_o) { assert(PyTuple_CheckExact(attrs_o)); attrs = PyStackRef_FromPyObjectSteal(attrs_o); @@ -9820,7 +9490,7 @@ } stack_pointer[0] = attrs; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -9847,7 +9517,7 @@ values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); stack_pointer[0] = values_or_none; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -9866,7 +9536,7 @@ res = match ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -9885,7 +9555,7 @@ res = match ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -9928,7 +9598,7 @@ ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -9946,7 +9616,7 @@ iter = stack_pointer[-2]; (void)index_or_null; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iter); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -9971,7 +9641,7 @@ RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -10014,7 +9684,7 @@ JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -10057,7 +9727,7 @@ JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -10079,7 +9749,7 @@ RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -10094,7 +9764,7 @@ _PyStackRef value; value = stack_pointer[-1]; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -10126,7 +9796,7 @@ stack_pointer[-1] = prev_exc; stack_pointer[0] = new_exc; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -10142,7 +9812,7 @@ res = PyStackRef_NULL; stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -10162,7 +9832,7 @@ PyObject *cause = oparg == 2 ? PyStackRef_AsPyObjectSteal(args[1]) : NULL; PyObject *exc = oparg > 0 ? PyStackRef_AsPyObjectSteal(args[0]) : NULL; stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int err = do_raise(tstate, exc, cause); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -10196,7 +9866,7 @@ } assert(exc && PyExceptionInstance_Check(exc)); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, this_instr); @@ -10365,7 +10035,7 @@ LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -10383,7 +10053,7 @@ assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); assert(STACK_LEVEL() == 0); _Py_LeaveRecursiveCallPy(tstate); @@ -10396,7 +10066,7 @@ LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -10445,7 +10115,7 @@ _PyInterpreterFrame *gen_frame = &gen->gi_iframe; _PyFrame_StackPush(gen_frame, PyStackRef_MakeHeapSafe(v)); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; @@ -10485,7 +10155,7 @@ } else { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -10493,7 +10163,7 @@ } } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -10501,7 +10171,7 @@ } stack_pointer[0] = retval; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -10561,7 +10231,7 @@ assert(tstate->interp->eval_frame == NULL); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); @@ -10640,7 +10310,7 @@ JUMP_TO_LABEL(pop_1_error); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -10668,7 +10338,7 @@ *ptr = attr; stack_pointer[-2] = func_out; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -10689,7 +10359,7 @@ PyStackRef_AsPyObjectBorrow(iterable)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -10748,7 +10418,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { JUMP_TO_LABEL(error); } @@ -10823,7 +10493,7 @@ } UNLOCK_OBJECT(owner_o); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); Py_XDECREF(old_value); @@ -10874,7 +10544,7 @@ FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); UNLOCK_OBJECT(owner_o); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); Py_XDECREF(old_value); @@ -10962,7 +10632,7 @@ UNLOCK_OBJECT(dict); STAT_INC(STORE_ATTR, hit); stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(owner); Py_XDECREF(old_value); @@ -10986,7 +10656,7 @@ PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -11003,7 +10673,7 @@ _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11050,14 +10720,14 @@ _PyStackRef tmp = GETLOCAL(oparg1); GETLOCAL(oparg1) = value1; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); tmp = GETLOCAL(oparg2); GETLOCAL(oparg2) = value2; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_XCLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11079,7 +10749,7 @@ int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11108,7 +10778,7 @@ "no locals found when storing %R", name); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11125,7 +10795,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11169,7 +10839,7 @@ } else { stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); Py_DECREF(slice); @@ -11187,7 +10857,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -4; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { JUMP_TO_LABEL(error); } @@ -11246,7 +10916,7 @@ PyStackRef_CLOSE(tmp); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { JUMP_TO_LABEL(error); } @@ -11294,7 +10964,7 @@ PyStackRef_AsPyObjectSteal(value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(dict_st); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11377,7 +11047,7 @@ UNLOCK_OBJECT(list); PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(list_st); Py_DECREF(old_value); @@ -11443,7 +11113,7 @@ int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11454,7 +11124,7 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -11489,7 +11159,7 @@ { value = owner; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11497,7 +11167,7 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -11554,7 +11224,7 @@ } else { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11671,7 +11341,7 @@ else { assert(Py_SIZE(value_o)); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11760,7 +11430,7 @@ PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11770,7 +11440,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -11789,7 +11459,7 @@ PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11799,7 +11469,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -11835,7 +11505,7 @@ top = &stack_pointer[(oparg & 0xFF) + (oparg >> 8)]; PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg & 0xFF, oparg >> 8, top); Py_DECREF(seq_o); @@ -11844,7 +11514,7 @@ JUMP_TO_LABEL(error); } stack_pointer += 1 + (oparg & 0xFF) + (oparg >> 8); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -11885,7 +11555,7 @@ top = &stack_pointer[-1 + oparg]; PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg, -1, top); Py_DECREF(seq_o); @@ -11895,7 +11565,7 @@ } } stack_pointer += oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -11950,7 +11620,7 @@ } UNLOCK_OBJECT(seq_o); stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(seq); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -12000,7 +11670,7 @@ *values++ = PyStackRef_FromPyObjectNew(items[i]); } stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(seq); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -12051,7 +11721,7 @@ stack_pointer[-1] = val1; stack_pointer[0] = val0; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(seq); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -12100,7 +11770,7 @@ res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -12123,7 +11793,7 @@ gen->gi_frame_state = FRAME_SUSPENDED + oparg; _PyStackRef temp = retval; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; @@ -12146,7 +11816,7 @@ LLTRACE_RESUME_FRAME(); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } diff --git a/Python/jit.c b/Python/jit.c index 7ab0f8ddd430dd..47d3d7a5d27180 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -60,6 +60,10 @@ jit_error(const char *message) static unsigned char * jit_alloc(size_t size) { + if (size > PY_MAX_JIT_CODE_SIZE) { + jit_error("code too big; refactor bytecodes.c to keep uop size down, or reduce maximum trace length."); + return NULL; + } assert(size); assert(size % get_page_size() == 0); #ifdef MS_WINDOWS diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 8d7b734e17cb0b..685659ef7c46ef 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -278,6 +278,36 @@ get_co_name(JitOptContext *ctx, int index) return PyTuple_GET_ITEM(get_current_code_object(ctx)->co_names, index); } +#ifdef Py_DEBUG +void +_Py_opt_assert_within_stack_bounds( + _Py_UOpsAbstractFrame *frame, JitOptRef *stack_pointer, + const char *filename, int lineno +) { + if (frame->code == ((PyCodeObject *)&_Py_InitCleanup)) { + return; + } + int level = (int)(stack_pointer - frame->stack); + if (level < 0) { + printf("Stack underflow (depth = %d) at %s:%d\n", level, filename, lineno); + fflush(stdout); + abort(); + } + int size = (int)(frame->stack_len); + if (level > size) { + printf("Stack overflow (depth = %d) at %s:%d\n", level, filename, lineno); + fflush(stdout); + abort(); + } +} +#endif + +#ifdef Py_DEBUG +#define ASSERT_WITHIN_STACK_BOUNDS(F, L) _Py_opt_assert_within_stack_bounds(ctx->frame, stack_pointer, (F), (L)) +#else +#define ASSERT_WITHIN_STACK_BOUNDS(F, L) (void)0 +#endif + // TODO (gh-134584) generate most of this table automatically const uint16_t op_without_decref_inputs[MAX_UOP_ID + 1] = { [_BINARY_OP_MULTIPLY_FLOAT] = _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS, diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 9ebd113df2dabf..0c2d34d2e640a3 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -35,7 +35,7 @@ } stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -44,7 +44,7 @@ value = GETLOCAL(oparg); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -53,7 +53,7 @@ value = PyJitRef_Borrow(GETLOCAL(oparg)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -64,7 +64,7 @@ GETLOCAL(oparg) = temp; stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -76,7 +76,7 @@ value = PyJitRef_Borrow(sym_new_const(ctx, val)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -89,7 +89,7 @@ value = PyJitRef_Borrow(sym_new_const(ctx, val)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -98,7 +98,7 @@ value = stack_pointer[-1]; GETLOCAL(oparg) = value; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -121,37 +121,37 @@ REPLACE_OP(this_instr, _POP_TOP_UNICODE, 0, 0); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_TOP_NOP: { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_TOP_INT: { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_TOP_FLOAT: { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_TOP_UNICODE: { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_TWO: { stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -160,19 +160,19 @@ res = sym_new_null(ctx); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _END_FOR: { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_ITER: { stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -181,7 +181,7 @@ val = sym_new_not_null(ctx); stack_pointer[-2] = val; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -506,13 +506,13 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } res = sym_new_compact_int(ctx); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -556,13 +556,13 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } res = sym_new_compact_int(ctx); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -606,13 +606,13 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } res = sym_new_compact_int(ctx); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -675,7 +675,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } res = sym_new_type(ctx, &PyFloat_Type); @@ -684,7 +684,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -727,7 +727,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } res = sym_new_type(ctx, &PyFloat_Type); @@ -736,7 +736,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -779,7 +779,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } res = sym_new_type(ctx, &PyFloat_Type); @@ -788,7 +788,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -797,7 +797,7 @@ res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -806,7 +806,7 @@ res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -815,7 +815,7 @@ res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -858,13 +858,13 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } res = sym_new_type(ctx, &PyUnicode_Type); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -889,7 +889,7 @@ } GETLOCAL(this_instr->operand0) = res; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -902,7 +902,7 @@ res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -922,13 +922,13 @@ } stack_pointer[-3] = res; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_SLICE: { stack_pointer += -4; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -937,7 +937,7 @@ res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -946,7 +946,7 @@ res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -955,7 +955,7 @@ res = sym_new_type(ctx, &PyUnicode_Type); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1004,7 +1004,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1033,7 +1033,7 @@ res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1042,7 +1042,7 @@ getitem = sym_new_not_null(ctx); stack_pointer[0] = getitem; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1052,43 +1052,43 @@ ctx->done = true; stack_pointer[-3] = new_frame; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _LIST_APPEND: { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _SET_ADD: { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_SUBSCR: { stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_SUBSCR_LIST_INT: { stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_SUBSCR_DICT: { stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _DELETE_SUBSCR: { stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1104,7 +1104,7 @@ res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1114,7 +1114,7 @@ retval = stack_pointer[-1]; JitOptRef temp = PyJitRef_StripReferenceInfo(retval); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); ctx->frame->stack_pointer = stack_pointer; PyCodeObject *returning_code = get_code_with_logging(this_instr); if (returning_code == NULL) { @@ -1136,7 +1136,7 @@ res = temp; stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1152,7 +1152,7 @@ awaitable = sym_new_not_null(ctx); stack_pointer[0] = awaitable; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1179,7 +1179,7 @@ retval = stack_pointer[-1]; JitOptRef temp = PyJitRef_StripReferenceInfo(retval); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); ctx->frame->stack_pointer = stack_pointer; PyCodeObject *returning_code = get_code_with_logging(this_instr); if (returning_code == NULL) { @@ -1195,13 +1195,13 @@ value = temp; stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_EXCEPT: { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1210,7 +1210,7 @@ value = sym_new_not_null(ctx); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1219,13 +1219,13 @@ bc = sym_new_not_null(ctx); stack_pointer[0] = bc; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_NAME: { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1243,7 +1243,7 @@ values[i] = sym_new_unknown(ctx); } stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1257,7 +1257,7 @@ stack_pointer[-1] = val1; stack_pointer[0] = val0; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1270,7 +1270,7 @@ values[i] = sym_tuple_getitem(ctx, seq, oparg - i - 1); } stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1281,7 +1281,7 @@ values[_i] = sym_new_not_null(ctx); } stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1296,25 +1296,25 @@ values[i] = sym_new_unknown(ctx); } stack_pointer += (oparg & 0xFF) + (oparg >> 8); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_ATTR: { stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _DELETE_ATTR: { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_GLOBAL: { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1327,7 +1327,7 @@ locals = sym_new_not_null(ctx); stack_pointer[0] = locals; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1338,7 +1338,7 @@ v = sym_new_not_null(ctx); stack_pointer[0] = v; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1347,7 +1347,7 @@ res = &stack_pointer[0]; res[0] = sym_new_not_null(ctx); stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1362,7 +1362,7 @@ REPLACE_OP(this_instr, _NOP, 0, 0); } stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1428,7 +1428,7 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1464,7 +1464,7 @@ } stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1492,13 +1492,13 @@ value = sym_new_not_null(ctx); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_DEREF: { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1511,7 +1511,7 @@ str = sym_new_type(ctx, &PyUnicode_Type); stack_pointer[-oparg] = str; stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1520,7 +1520,7 @@ interpolation = sym_new_not_null(ctx); stack_pointer[-2 - (oparg & 1)] = interpolation; stack_pointer += -1 - (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1529,7 +1529,7 @@ template = sym_new_not_null(ctx); stack_pointer[-2] = template; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1540,7 +1540,7 @@ tup = sym_new_tuple(ctx, oparg, values); stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1549,19 +1549,19 @@ list = sym_new_type(ctx, &PyList_Type); stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _LIST_EXTEND: { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _SET_UPDATE: { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1570,7 +1570,7 @@ set = sym_new_type(ctx, &PySet_Type); stack_pointer[-oparg] = set; stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1579,7 +1579,7 @@ map = sym_new_type(ctx, &PyDict_Type); stack_pointer[-oparg*2] = map; stack_pointer += 1 - oparg*2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1589,19 +1589,19 @@ case _DICT_UPDATE: { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _DICT_MERGE: { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _MAP_ADD: { stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1610,7 +1610,7 @@ attr_st = sym_new_not_null(ctx); stack_pointer[-3] = attr_st; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1622,7 +1622,7 @@ stack_pointer[-3] = attr; stack_pointer[-2] = self_or_null; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1639,7 +1639,7 @@ self_or_null[0] = sym_new_unknown(ctx); } stack_pointer += (oparg&1); - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1781,19 +1781,19 @@ case _STORE_ATTR_INSTANCE_VALUE: { stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_ATTR_WITH_HINT: { stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_ATTR_SLOT: { stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1842,7 +1842,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } if (oparg & 16) { @@ -1853,7 +1853,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1893,13 +1893,13 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } res = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1943,13 +1943,13 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } res = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1991,13 +1991,13 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } res = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2006,7 +2006,7 @@ b = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = b; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2044,13 +2044,13 @@ } stack_pointer[-2] = b; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } b = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = b; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2070,7 +2070,7 @@ b = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = b; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2079,7 +2079,7 @@ b = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = b; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2105,7 +2105,7 @@ res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2114,7 +2114,7 @@ res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2151,13 +2151,13 @@ len = sym_new_const(ctx, temp); stack_pointer[0] = len; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); Py_DECREF(temp); stack_pointer += -1; } stack_pointer[0] = len; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2166,7 +2166,7 @@ attrs = sym_new_not_null(ctx); stack_pointer[-3] = attrs; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2175,7 +2175,7 @@ res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2184,7 +2184,7 @@ res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2193,7 +2193,7 @@ values_or_none = sym_new_not_null(ctx); stack_pointer[0] = values_or_none; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2213,7 +2213,7 @@ stack_pointer[-1] = iter; stack_pointer[0] = index_or_null; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2231,7 +2231,7 @@ next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2254,7 +2254,7 @@ next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2279,7 +2279,7 @@ next = sym_new_not_null(ctx); stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2298,7 +2298,7 @@ next = sym_new_type(ctx, &PyLong_Type); stack_pointer[0] = next; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2308,7 +2308,7 @@ ctx->done = true; stack_pointer[0] = gen_frame; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2320,7 +2320,7 @@ method_and_self[0] = sym_new_null(ctx); method_and_self[1] = self; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2337,7 +2337,7 @@ res = sym_new_not_null(ctx); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2349,7 +2349,7 @@ stack_pointer[-1] = prev_exc; stack_pointer[0] = new_exc; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2377,7 +2377,7 @@ stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2397,7 +2397,7 @@ stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2451,7 +2451,7 @@ stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2485,7 +2485,7 @@ new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2533,7 +2533,7 @@ res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2618,7 +2618,7 @@ } stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2626,7 +2626,7 @@ JitOptRef new_frame; new_frame = stack_pointer[-1]; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (!CURRENT_FRAME_IS_INIT_SHIM()) { ctx->frame->stack_pointer = stack_pointer; } @@ -2705,7 +2705,7 @@ } stack_pointer[-3] = res; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2731,7 +2731,7 @@ } stack_pointer[-3] = res; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2757,7 +2757,7 @@ } stack_pointer[-3] = res; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2799,13 +2799,13 @@ init_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, args-1, oparg+1)); stack_pointer[-2 - oparg] = init_frame; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _EXIT_INIT_CHECK: { stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2814,7 +2814,7 @@ res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2823,7 +2823,7 @@ res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2832,7 +2832,7 @@ res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2841,7 +2841,7 @@ res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2874,13 +2874,13 @@ res = sym_new_const(ctx, temp); stack_pointer[-3] = res; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); Py_DECREF(temp); stack_pointer += 2; } stack_pointer[-3] = res; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2914,7 +2914,7 @@ } stack_pointer[-4] = res; stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2931,7 +2931,7 @@ case _CALL_LIST_APPEND: { stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2940,7 +2940,7 @@ res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2949,7 +2949,7 @@ res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2958,7 +2958,7 @@ res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2967,7 +2967,7 @@ res = sym_new_not_null(ctx); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2990,7 +2990,7 @@ new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); stack_pointer[-3 - oparg] = new_frame; stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3015,7 +3015,7 @@ res = sym_new_not_null(ctx); stack_pointer[-3 - oparg] = res; stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3037,7 +3037,7 @@ func_out = sym_new_not_null(ctx); stack_pointer[-2] = func_out; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3058,7 +3058,7 @@ res = sym_new_unknown(ctx); stack_pointer[0] = res; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3067,7 +3067,7 @@ slice = sym_new_type(ctx, &PySlice_Type); stack_pointer[-oparg] = slice; stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3090,7 +3090,7 @@ res = sym_new_not_null(ctx); stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3102,7 +3102,7 @@ top = bottom; stack_pointer[0] = top; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3141,7 +3141,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } bool lhs_int = sym_matches_type(lhs, &PyLong_Type); @@ -3179,7 +3179,7 @@ } stack_pointer[-2] = res; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3225,7 +3225,7 @@ } sym_set_const(flag, Py_True); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3239,7 +3239,7 @@ } sym_set_const(flag, Py_False); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3257,7 +3257,7 @@ } sym_set_const(val, Py_None); stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3274,7 +3274,7 @@ eliminate_pop_guard(this_instr, false); } stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3319,7 +3319,7 @@ value = sym_new_const(ctx, ptr); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3337,25 +3337,25 @@ value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[0] = value; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_CALL: { stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_CALL_ONE: { stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_CALL_TWO: { stack_pointer += -4; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3372,7 +3372,7 @@ value = sym_new_not_null(ctx); stack_pointer[-2] = value; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3382,7 +3382,7 @@ value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[-2] = value; stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3392,7 +3392,7 @@ value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[-3] = value; stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3402,7 +3402,7 @@ value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); stack_pointer[-4] = value; stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3414,7 +3414,7 @@ stack_pointer[-1] = value; stack_pointer[0] = new; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -3426,7 +3426,7 @@ stack_pointer[-1] = value; stack_pointer[0] = new; stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index 3a0e7e5d0d5636..6519e8e4f3ed36 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -296,7 +296,7 @@ def _save_physical_sp(self, out: CWriter) -> None: diff = self.logical_sp - self.physical_sp out.start_line() out.emit(f"stack_pointer += {diff.to_c()};\n") - out.emit(f"assert(WITHIN_STACK_BOUNDS());\n") + out.emit(f"ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__);\n") self.physical_sp = self.logical_sp self._print(out) diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 857e926d119900..0167f1b0ae5e37 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -86,6 +86,12 @@ do { \ #define TIER_TWO 2 +#ifdef Py_DEBUG +#define ASSERT_WITHIN_STACK_BOUNDS(F, L) _Py_assert_within_stack_bounds(frame, stack_pointer, (F), (L)) +#else +#define ASSERT_WITHIN_STACK_BOUNDS(F, L) (void)0 +#endif + __attribute__((preserve_none)) _Py_CODEUNIT * _JIT_ENTRY(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate) { From c0c65141b37029bfb364094a6dfb4c75ebf8359e Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Wed, 3 Dec 2025 15:48:44 -0500 Subject: [PATCH 249/638] gh-140482: Avoid changing terminal settings in test_pty (gh-142202) The previous test_spawn_doesnt_hang test had a few problems: * It would cause ENV CHANGED failures if other tests were running concurrently due to stty changes * Typing while the test was running could cause it to fail --- Lib/test/test_pty.py | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/Lib/test/test_pty.py b/Lib/test/test_pty.py index a2018e864445e1..7e4f4828ce0f8d 100644 --- a/Lib/test/test_pty.py +++ b/Lib/test/test_pty.py @@ -3,7 +3,6 @@ is_android, is_apple_mobile, is_wasm32, reap_children, verbose, warnings_helper ) from test.support.import_helper import import_module -from test.support.os_helper import TESTFN, unlink # Skip these tests if termios is not available import_module('termios') @@ -299,26 +298,27 @@ def test_master_read(self): @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_spawn_doesnt_hang(self): - self.addCleanup(unlink, TESTFN) - with open(TESTFN, 'wb') as f: - STDOUT_FILENO = 1 - dup_stdout = os.dup(STDOUT_FILENO) - os.dup2(f.fileno(), STDOUT_FILENO) - buf = b'' - def master_read(fd): - nonlocal buf - data = os.read(fd, 1024) - buf += data - return data + # gh-140482: Do the test in a pty.fork() child to avoid messing + # with the interactive test runner's terminal settings. + pid, fd = pty.fork() + if pid == pty.CHILD: + pty.spawn([sys.executable, '-c', 'print("hi there")']) + os._exit(0) + + try: + buf = bytearray() try: - pty.spawn([sys.executable, '-c', 'print("hi there")'], - master_read) - finally: - os.dup2(dup_stdout, STDOUT_FILENO) - os.close(dup_stdout) - self.assertEqual(buf, b'hi there\r\n') - with open(TESTFN, 'rb') as f: - self.assertEqual(f.read(), b'hi there\r\n') + while (data := os.read(fd, 1024)) != b'': + buf.extend(data) + except OSError as e: + if e.errno != errno.EIO: + raise + + (pid, status) = os.waitpid(pid, 0) + self.assertEqual(status, 0) + self.assertEqual(buf.take_bytes(), b"hi there\r\n") + finally: + os.close(fd) class SmallPtyTests(unittest.TestCase): """These tests don't spawn children or hang.""" From 618dc367146069f8f0aaeb0a4a7f1b834dc4a213 Mon Sep 17 00:00:00 2001 From: Chris Eibl <138194463+chris-eibl@users.noreply.github.com> Date: Wed, 3 Dec 2025 23:08:51 +0100 Subject: [PATCH 250/638] GH-142050: Jit stencils on Windows contain debug data (#142052) Co-authored-by: Savannah Ostrowski --- .../next/Build/2025-11-28-21-43-07.gh-issue-142050.PFi4tv.rst | 1 + Tools/jit/_schema.py | 1 + Tools/jit/_targets.py | 4 ++++ 3 files changed, 6 insertions(+) create mode 100644 Misc/NEWS.d/next/Build/2025-11-28-21-43-07.gh-issue-142050.PFi4tv.rst diff --git a/Misc/NEWS.d/next/Build/2025-11-28-21-43-07.gh-issue-142050.PFi4tv.rst b/Misc/NEWS.d/next/Build/2025-11-28-21-43-07.gh-issue-142050.PFi4tv.rst new file mode 100644 index 00000000000000..8917d5df76e5c0 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-11-28-21-43-07.gh-issue-142050.PFi4tv.rst @@ -0,0 +1 @@ +Fixed a bug where JIT stencils produced on Windows contained debug data. Patch by Chris Eibl. diff --git a/Tools/jit/_schema.py b/Tools/jit/_schema.py index c47e9af924a20e..4e86abe604972e 100644 --- a/Tools/jit/_schema.py +++ b/Tools/jit/_schema.py @@ -89,6 +89,7 @@ class COFFSection(typing.TypedDict): Characteristics: dict[ typing.Literal["Flags"], list[dict[typing.Literal["Name"], str]] ] + Name: dict[typing.Literal["Value"], str] Number: int RawDataSize: int Relocations: list[dict[typing.Literal["Relocation"], COFFRelocation]] diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index a76d8ff2792602..4c188d74a68602 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -267,6 +267,10 @@ class _COFF( def _handle_section( self, section: _schema.COFFSection, group: _stencils.StencilGroup ) -> None: + name = section["Name"]["Value"] + if name == ".debug$S": + # skip debug sections + return flags = {flag["Name"] for flag in section["Characteristics"]["Flags"]} if "SectionData" in section: section_data_bytes = section["SectionData"]["Bytes"] From 547d8daf780646e2800bec598ed32085817c8606 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Wed, 3 Dec 2025 18:37:35 -0500 Subject: [PATCH 251/638] gh-142218: Fix split table dictionary crash (gh-142229) This fixes a regression introduced in gh-140558. The interpreter would crash if we inserted a non `str` key into a split table that matches an existing key. --- Lib/test/test_dict.py | 8 ++++++++ .../2025-12-03-11-03-35.gh-issue-142218.44Fq_J.rst | 2 ++ Objects/dictobject.c | 10 +++++++--- 3 files changed, 17 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-12-03-11-03-35.gh-issue-142218.44Fq_J.rst diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 2e6c2bbdf19409..665b3e843dd3a5 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1621,6 +1621,14 @@ def __eq__(self, other): self.assertEqual(len(d), 1) + def test_split_table_update_with_str_subclass(self): + class MyStr(str): pass + class MyClass: pass + obj = MyClass() + obj.attr = 1 + obj.__dict__[MyStr('attr')] = 2 + self.assertEqual(obj.attr, 2) + class CAPITest(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-12-03-11-03-35.gh-issue-142218.44Fq_J.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-03-11-03-35.gh-issue-142218.44Fq_J.rst new file mode 100644 index 00000000000000..a8ce0fc65267d5 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-03-11-03-35.gh-issue-142218.44Fq_J.rst @@ -0,0 +1,2 @@ +Fix crash when inserting into a split table dictionary with a non +:class:`str` key that matches an existing key. diff --git a/Objects/dictobject.c b/Objects/dictobject.c index ee1c173ae4abb0..e0eef7b46df4b2 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -1914,10 +1914,14 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, if (old_value != value) { _PyDict_NotifyEvent(interp, PyDict_EVENT_MODIFIED, mp, key, value); assert(old_value != NULL); - assert(!_PyDict_HasSplitTable(mp)); if (DK_IS_UNICODE(mp->ma_keys)) { - PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[ix]; - STORE_VALUE(ep, value); + if (_PyDict_HasSplitTable(mp)) { + STORE_SPLIT_VALUE(mp, ix, value); + } + else { + PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[ix]; + STORE_VALUE(ep, value); + } } else { PyDictKeyEntry *ep = &DK_ENTRIES(mp->ma_keys)[ix]; From c5252045d3a7164f1829503d122091b5e469fda3 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Wed, 3 Dec 2025 15:42:10 -0800 Subject: [PATCH 252/638] Being more flexible in when not to explicitly set the sysroot when compiling for WASI (GH-142242) --- Tools/wasm/wasi/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/wasm/wasi/__main__.py b/Tools/wasm/wasi/__main__.py index 06903fd25abe44..d95cc99c8ea28b 100644 --- a/Tools/wasm/wasi/__main__.py +++ b/Tools/wasm/wasi/__main__.py @@ -271,7 +271,7 @@ def wasi_sdk_env(context): for env_var, binary_name in list(env.items()): env[env_var] = os.fsdecode(wasi_sdk_path / "bin" / binary_name) - if wasi_sdk_path != pathlib.Path("/opt/wasi-sdk"): + if not wasi_sdk_path.name.startswith("wasi-sdk"): for compiler in ["CC", "CPP", "CXX"]: env[compiler] += f" --sysroot={sysroot}" From 1a7824a927f0706300af7bfc182884a43e2f587a Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Wed, 3 Dec 2025 22:14:25 -0500 Subject: [PATCH 253/638] gh-141004: Add a CI job ensuring that new C APIs include documentation (GH-142102) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- .github/CODEOWNERS | 3 + .github/workflows/build.yml | 3 + Makefile.pre.in | 5 + Tools/check-c-api-docs/ignored_c_api.txt | 93 +++++++++++ Tools/check-c-api-docs/main.py | 193 +++++++++++++++++++++++ 5 files changed, 297 insertions(+) create mode 100644 Tools/check-c-api-docs/ignored_c_api.txt create mode 100644 Tools/check-c-api-docs/main.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1086b42620479d..6acc156ebff713 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -126,6 +126,9 @@ Doc/howto/clinic.rst @erlend-aasland @AA-Turner # C Analyser Tools/c-analyzer/ @ericsnowcurrently +# C API Documentation Checks +Tools/check-c-api-docs/ @ZeroIntensity + # Fuzzing Modules/_xxtestfuzz/ @ammaraskar diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8e15400e4978eb..3d889fa128e261 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -142,6 +142,9 @@ jobs: - name: Check for unsupported C global variables if: github.event_name == 'pull_request' # $GITHUB_EVENT_NAME run: make check-c-globals + - name: Check for undocumented C APIs + run: make check-c-api-docs + build-windows: name: >- diff --git a/Makefile.pre.in b/Makefile.pre.in index 086adbdf262c48..f3086ec1462b6b 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -3322,6 +3322,11 @@ check-c-globals: --format summary \ --traceback +# Check for undocumented C APIs. +.PHONY: check-c-api-docs +check-c-api-docs: + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/check-c-api-docs/main.py + # Find files with funny names .PHONY: funny funny: diff --git a/Tools/check-c-api-docs/ignored_c_api.txt b/Tools/check-c-api-docs/ignored_c_api.txt new file mode 100644 index 00000000000000..e81ffd51e193b2 --- /dev/null +++ b/Tools/check-c-api-docs/ignored_c_api.txt @@ -0,0 +1,93 @@ +# pydtrace_probes.h +PyDTrace_AUDIT +PyDTrace_FUNCTION_ENTRY +PyDTrace_FUNCTION_RETURN +PyDTrace_GC_DONE +PyDTrace_GC_START +PyDTrace_IMPORT_FIND_LOAD_DONE +PyDTrace_IMPORT_FIND_LOAD_START +PyDTrace_INSTANCE_DELETE_DONE +PyDTrace_INSTANCE_DELETE_START +PyDTrace_INSTANCE_NEW_DONE +PyDTrace_INSTANCE_NEW_START +PyDTrace_LINE +# fileobject.h +Py_FileSystemDefaultEncodeErrors +Py_FileSystemDefaultEncoding +Py_HasFileSystemDefaultEncoding +Py_UTF8Mode +# pyhash.h +Py_HASH_EXTERNAL +# exports.h +PyAPI_DATA +Py_EXPORTED_SYMBOL +Py_IMPORTED_SYMBOL +Py_LOCAL_SYMBOL +# modsupport.h +PyABIInfo_FREETHREADING_AGNOSTIC +# moduleobject.h +PyModuleDef_Type +# object.h +Py_INVALID_SIZE +Py_TPFLAGS_HAVE_VERSION_TAG +Py_TPFLAGS_INLINE_VALUES +Py_TPFLAGS_IS_ABSTRACT +# pyexpat.h +PyExpat_CAPI_MAGIC +PyExpat_CAPSULE_NAME +# pyport.h +Py_ALIGNED +Py_ARITHMETIC_RIGHT_SHIFT +Py_CAN_START_THREADS +Py_FORCE_EXPANSION +Py_GCC_ATTRIBUTE +Py_LL +Py_SAFE_DOWNCAST +Py_ULL +Py_VA_COPY +# unicodeobject.h +Py_UNICODE_SIZE +# cpython/methodobject.h +PyCFunction_GET_CLASS +# cpython/compile.h +PyCF_ALLOW_INCOMPLETE_INPUT +PyCF_COMPILE_MASK +PyCF_DONT_IMPLY_DEDENT +PyCF_IGNORE_COOKIE +PyCF_MASK +PyCF_MASK_OBSOLETE +PyCF_SOURCE_IS_UTF8 +# cpython/descrobject.h +PyDescr_COMMON +PyDescr_NAME +PyDescr_TYPE +PyWrapperFlag_KEYWORDS +# cpython/fileobject.h +PyFile_NewStdPrinter +PyStdPrinter_Type +Py_UniversalNewlineFgets +# cpython/setobject.h +PySet_MINSIZE +# cpython/ceval.h +PyUnstable_CopyPerfMapFile +PyUnstable_PerfTrampoline_CompileCode +PyUnstable_PerfTrampoline_SetPersistAfterFork +# cpython/genobject.h +PyAsyncGenASend_CheckExact +# cpython/longintrepr.h +PyLong_BASE +PyLong_MASK +PyLong_SHIFT +# cpython/pyerrors.h +PyException_HEAD +# cpython/pyframe.h +PyUnstable_EXECUTABLE_KINDS +PyUnstable_EXECUTABLE_KIND_BUILTIN_FUNCTION +PyUnstable_EXECUTABLE_KIND_METHOD_DESCRIPTOR +PyUnstable_EXECUTABLE_KIND_PY_FUNCTION +PyUnstable_EXECUTABLE_KIND_SKIP +# cpython/pylifecycle.h +Py_FrozenMain +# cpython/unicodeobject.h +PyUnicode_IS_COMPACT +PyUnicode_IS_COMPACT_ASCII diff --git a/Tools/check-c-api-docs/main.py b/Tools/check-c-api-docs/main.py new file mode 100644 index 00000000000000..6bdf80a9ae8985 --- /dev/null +++ b/Tools/check-c-api-docs/main.py @@ -0,0 +1,193 @@ +import re +from pathlib import Path +import sys +import _colorize +import textwrap + +SIMPLE_FUNCTION_REGEX = re.compile(r"PyAPI_FUNC(.+) (\w+)\(") +SIMPLE_MACRO_REGEX = re.compile(r"# *define *(\w+)(\(.+\))? ") +SIMPLE_INLINE_REGEX = re.compile(r"static inline .+( |\n)(\w+)") +SIMPLE_DATA_REGEX = re.compile(r"PyAPI_DATA\(.+\) (\w+)") + +CPYTHON = Path(__file__).parent.parent.parent +INCLUDE = CPYTHON / "Include" +C_API_DOCS = CPYTHON / "Doc" / "c-api" +IGNORED = ( + (CPYTHON / "Tools" / "check-c-api-docs" / "ignored_c_api.txt") + .read_text() + .split("\n") +) + +for index, line in enumerate(IGNORED): + if line.startswith("#"): + IGNORED.pop(index) + +MISTAKE = """ +If this is a mistake and this script should not be failing, create an +issue and tag Peter (@ZeroIntensity) on it.\ +""" + + +def found_undocumented(singular: bool) -> str: + some = "an" if singular else "some" + s = "" if singular else "s" + these = "this" if singular else "these" + them = "it" if singular else "them" + were = "was" if singular else "were" + + return ( + textwrap.dedent( + f""" + Found {some} undocumented C API{s}! + + Python requires documentation on all public C API symbols, macros, and types. + If {these} API{s} {were} not meant to be public, prefix {them} with a + leading underscore (_PySomething_API) or move {them} to the internal C API + (pycore_*.h files). + + In exceptional cases, certain APIs can be ignored by adding them to + Tools/check-c-api-docs/ignored_c_api.txt + """ + ) + + MISTAKE + ) + + +def found_ignored_documented(singular: bool) -> str: + some = "a" if singular else "some" + s = "" if singular else "s" + them = "it" if singular else "them" + were = "was" if singular else "were" + they = "it" if singular else "they" + + return ( + textwrap.dedent( + f""" + Found {some} C API{s} listed in Tools/c-api-docs-check/ignored_c_api.txt, but + {they} {were} found in the documentation. To fix this, remove {them} from + ignored_c_api.txt. + """ + ) + + MISTAKE + ) + + +def is_documented(name: str) -> bool: + """ + Is a name present in the C API documentation? + """ + for path in C_API_DOCS.iterdir(): + if path.is_dir(): + continue + if path.suffix != ".rst": + continue + + text = path.read_text(encoding="utf-8") + if name in text: + return True + + return False + + +def scan_file_for_docs(filename: str, text: str) -> tuple[list[str], list[str]]: + """ + Scan a header file for C API functions. + """ + undocumented: list[str] = [] + documented_ignored: list[str] = [] + colors = _colorize.get_colors() + + def check_for_name(name: str) -> None: + documented = is_documented(name) + if documented and (name in IGNORED): + documented_ignored.append(name) + elif not documented and (name not in IGNORED): + undocumented.append(name) + + for function in SIMPLE_FUNCTION_REGEX.finditer(text): + name = function.group(2) + if not name.startswith("Py"): + continue + + check_for_name(name) + + for macro in SIMPLE_MACRO_REGEX.finditer(text): + name = macro.group(1) + if not name.startswith("Py"): + continue + + if "(" in name: + name = name[: name.index("(")] + + check_for_name(name) + + for inline in SIMPLE_INLINE_REGEX.finditer(text): + name = inline.group(2) + if not name.startswith("Py"): + continue + + check_for_name(name) + + for data in SIMPLE_DATA_REGEX.finditer(text): + name = data.group(1) + if not name.startswith("Py"): + continue + + check_for_name(name) + + # Remove duplicates and sort alphabetically to keep the output deterministic + undocumented = list(set(undocumented)) + undocumented.sort() + + if undocumented or documented_ignored: + print(f"{filename} {colors.RED}BAD{colors.RESET}") + for name in undocumented: + print(f"{colors.BOLD_RED}UNDOCUMENTED:{colors.RESET} {name}") + for name in documented_ignored: + print(f"{colors.BOLD_YELLOW}DOCUMENTED BUT IGNORED:{colors.RESET} {name}") + else: + print(f"{filename} {colors.GREEN}OK{colors.RESET}") + + return undocumented, documented_ignored + + +def main() -> None: + print("Scanning for undocumented C API functions...") + files = [*INCLUDE.iterdir(), *(INCLUDE / "cpython").iterdir()] + all_missing: list[str] = [] + all_found_ignored: list[str] = [] + + for file in files: + if file.is_dir(): + continue + assert file.exists() + text = file.read_text(encoding="utf-8") + missing, ignored = scan_file_for_docs(str(file.relative_to(INCLUDE)), text) + all_found_ignored += ignored + all_missing += missing + + fail = False + to_check = [ + (all_missing, "missing", found_undocumented(len(all_missing) == 1)), + ( + all_found_ignored, + "documented but ignored", + found_ignored_documented(len(all_found_ignored) == 1), + ), + ] + for name_list, what, message in to_check: + if not name_list: + continue + + s = "s" if len(name_list) != 1 else "" + print(f"-- {len(name_list)} {what} C API{s} --") + for name in name_list: + print(f" - {name}") + print(message) + fail = True + + sys.exit(1 if fail else 0) + + +if __name__ == "__main__": + main() From fb404ab575de7ab7658069810b7d771fb70d9fe4 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Thu, 4 Dec 2025 09:33:15 +0000 Subject: [PATCH 254/638] gh-142225: Fix `PyABIInfo_VAR` macro (GH-142230) --- Include/modsupport.h | 2 +- .../next/C_API/2025-12-03-16-35-24.gh-issue-142225.vmCJoo.rst | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-12-03-16-35-24.gh-issue-142225.vmCJoo.rst diff --git a/Include/modsupport.h b/Include/modsupport.h index 094b9ff0e5ccf8..cb47ad8cd2727f 100644 --- a/Include/modsupport.h +++ b/Include/modsupport.h @@ -132,7 +132,7 @@ PyAPI_FUNC(int) PyABIInfo_Check(PyABIInfo *info, const char *module_name); ) \ ///////////////////////////////////////////////////////// -#define _PyABIInfo_DEFAULT() { \ +#define _PyABIInfo_DEFAULT { \ 1, 0, \ PyABIInfo_DEFAULT_FLAGS, \ PY_VERSION_HEX, \ diff --git a/Misc/NEWS.d/next/C_API/2025-12-03-16-35-24.gh-issue-142225.vmCJoo.rst b/Misc/NEWS.d/next/C_API/2025-12-03-16-35-24.gh-issue-142225.vmCJoo.rst new file mode 100644 index 00000000000000..1eaf5b713d9cf5 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-12-03-16-35-24.gh-issue-142225.vmCJoo.rst @@ -0,0 +1 @@ +Fixed the :c:macro:`PyABIInfo_VAR` macro. From 6825d5c11ddb1dac7602fde55f0ed64e1aab50e7 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 4 Dec 2025 12:27:15 +0000 Subject: [PATCH 255/638] GH-139757: Fix reference leaks introduced in GH-140800 (GH-142257) --- Python/ceval.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index 1709dda0cbe145..46bf644106ac39 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1228,6 +1228,8 @@ _Py_BuildString_StackRefSteal( goto cleanup; } res = _PyUnicode_JoinArray(&_Py_STR(empty), args_o, total_args); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res != NULL) ^ (PyErr_Occurred() != NULL)); cleanup: // arguments is a pointer into the GC visible stack, // so we must NULL out values as we clear them. @@ -1239,8 +1241,6 @@ _Py_BuildString_StackRefSteal( return res; } - - PyObject * _Py_BuildMap_StackRefSteal( _PyStackRef *arguments, @@ -1257,6 +1257,8 @@ _Py_BuildMap_StackRefSteal( args_o+1, 2, half_args ); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res != NULL) ^ (PyErr_Occurred() != NULL)); cleanup: // arguments is a pointer into the GC visible stack, // so we must NULL out values as we clear them. From 8392095bf969655faf785dd0932c3f02fc4ec311 Mon Sep 17 00:00:00 2001 From: Kir Chou <148194051+gkirchou@users.noreply.github.com> Date: Thu, 4 Dec 2025 22:32:23 +0900 Subject: [PATCH 256/638] gh-129483: Make `TestLocalTimeDisambiguation`'s time format locale independent (#142193) * Change to update %c to the exact time format. --------- Co-authored-by: Kir Chou --- Lib/test/datetimetester.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 7df27206206268..ace56aab7aceba 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -6300,21 +6300,21 @@ def test_vilnius_1941_fromutc(self): gdt = datetime(1941, 6, 23, 20, 59, 59, tzinfo=timezone.utc) ldt = gdt.astimezone(Vilnius) - self.assertEqual(ldt.strftime("%c %Z%z"), + self.assertEqual(ldt.strftime("%a %b %d %H:%M:%S %Y %Z%z"), 'Mon Jun 23 23:59:59 1941 MSK+0300') self.assertEqual(ldt.fold, 0) self.assertFalse(ldt.dst()) gdt = datetime(1941, 6, 23, 21, tzinfo=timezone.utc) ldt = gdt.astimezone(Vilnius) - self.assertEqual(ldt.strftime("%c %Z%z"), + self.assertEqual(ldt.strftime("%a %b %d %H:%M:%S %Y %Z%z"), 'Mon Jun 23 23:00:00 1941 CEST+0200') self.assertEqual(ldt.fold, 1) self.assertTrue(ldt.dst()) gdt = datetime(1941, 6, 23, 22, tzinfo=timezone.utc) ldt = gdt.astimezone(Vilnius) - self.assertEqual(ldt.strftime("%c %Z%z"), + self.assertEqual(ldt.strftime("%a %b %d %H:%M:%S %Y %Z%z"), 'Tue Jun 24 00:00:00 1941 CEST+0200') self.assertEqual(ldt.fold, 0) self.assertTrue(ldt.dst()) @@ -6324,22 +6324,22 @@ def test_vilnius_1941_toutc(self): ldt = datetime(1941, 6, 23, 22, 59, 59, tzinfo=Vilnius) gdt = ldt.astimezone(timezone.utc) - self.assertEqual(gdt.strftime("%c %Z"), + self.assertEqual(gdt.strftime("%a %b %d %H:%M:%S %Y %Z"), 'Mon Jun 23 19:59:59 1941 UTC') ldt = datetime(1941, 6, 23, 23, 59, 59, tzinfo=Vilnius) gdt = ldt.astimezone(timezone.utc) - self.assertEqual(gdt.strftime("%c %Z"), + self.assertEqual(gdt.strftime("%a %b %d %H:%M:%S %Y %Z"), 'Mon Jun 23 20:59:59 1941 UTC') ldt = datetime(1941, 6, 23, 23, 59, 59, tzinfo=Vilnius, fold=1) gdt = ldt.astimezone(timezone.utc) - self.assertEqual(gdt.strftime("%c %Z"), + self.assertEqual(gdt.strftime("%a %b %d %H:%M:%S %Y %Z"), 'Mon Jun 23 21:59:59 1941 UTC') ldt = datetime(1941, 6, 24, 0, tzinfo=Vilnius) gdt = ldt.astimezone(timezone.utc) - self.assertEqual(gdt.strftime("%c %Z"), + self.assertEqual(gdt.strftime("%a %b %d %H:%M:%S %Y %Z"), 'Mon Jun 23 22:00:00 1941 UTC') def test_constructors(self): From 2dac9e6016c81abbefa4256253ff5c59b29378a7 Mon Sep 17 00:00:00 2001 From: Alper Date: Thu, 4 Dec 2025 06:21:51 -0800 Subject: [PATCH 257/638] gh-116738: Statically initialize special constants in cmath module (gh-142161) The initialization during `mod_exec` wasn't thread-safe with multiple interpreters. --- ...-12-01-10-03-08.gh-issue-116738.972YsG.rst | 2 + Modules/cmathmodule.c | 225 ++++++++---------- 2 files changed, 101 insertions(+), 126 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-12-01-10-03-08.gh-issue-116738.972YsG.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-12-01-10-03-08.gh-issue-116738.972YsG.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-01-10-03-08.gh-issue-116738.972YsG.rst new file mode 100644 index 00000000000000..d6d9d02b017473 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-01-10-03-08.gh-issue-116738.972YsG.rst @@ -0,0 +1,2 @@ +Fix :mod:`cmath` data race when initializing trigonometric tables with +subinterpreters. diff --git a/Modules/cmathmodule.c b/Modules/cmathmodule.c index aee3e4f343d8be..65fbcf5cdaa73f 100644 --- a/Modules/cmathmodule.c +++ b/Modules/cmathmodule.c @@ -163,8 +163,15 @@ special_type(double d) raised. */ -static Py_complex acos_special_values[7][7]; - +static Py_complex acos_special_values[7][7] = { + { {P34,INF}, {P,INF}, {P,INF}, {P,-INF}, {P,-INF}, {P34,-INF}, {N,INF} }, + { {P12,INF}, {U,U}, {U,U}, {U,U}, {U,U}, {P12,-INF}, {N,N} }, + { {P12,INF}, {U,U}, {P12,0.}, {P12,-0.}, {U,U}, {P12,-INF}, {P12,N} }, + { {P12,INF}, {U,U}, {P12,0.}, {P12,-0.}, {U,U}, {P12,-INF}, {P12,N} }, + { {P12,INF}, {U,U}, {U,U}, {U,U}, {U,U}, {P12,-INF}, {N,N} }, + { {P14,INF}, {0.,INF}, {0.,INF}, {0.,-INF}, {0.,-INF}, {P14,-INF}, {N,INF} }, + { {N,INF}, {N,N}, {N,N}, {N,N}, {N,N}, {N,-INF}, {N,N} } +}; /*[clinic input] cmath.acos -> Py_complex_protected @@ -202,7 +209,15 @@ cmath_acos_impl(PyObject *module, Py_complex z) } -static Py_complex acosh_special_values[7][7]; +static Py_complex acosh_special_values[7][7] = { + { {INF,-P34}, {INF,-P}, {INF,-P}, {INF,P}, {INF,P}, {INF,P34}, {INF,N} }, + { {INF,-P12}, {U,U}, {U,U}, {U,U}, {U,U}, {INF,P12}, {N,N} }, + { {INF,-P12}, {U,U}, {0.,-P12}, {0.,P12}, {U,U}, {INF,P12}, {N,P12} }, + { {INF,-P12}, {U,U}, {0.,-P12}, {0.,P12}, {U,U}, {INF,P12}, {N,P12} }, + { {INF,-P12}, {U,U}, {U,U}, {U,U}, {U,U}, {INF,P12}, {N,N} }, + { {INF,-P14}, {INF,-0.}, {INF,-0.}, {INF,0.}, {INF,0.}, {INF,P14}, {INF,N} }, + { {INF,N}, {N,N}, {N,N}, {N,N}, {N,N}, {INF,N}, {N,N} } +}; /*[clinic input] cmath.acosh = cmath.acos @@ -257,7 +272,15 @@ cmath_asin_impl(PyObject *module, Py_complex z) } -static Py_complex asinh_special_values[7][7]; +static Py_complex asinh_special_values[7][7] = { + { {-INF,-P14}, {-INF,-0.}, {-INF,-0.}, {-INF,0.}, {-INF,0.}, {-INF,P14}, {-INF,N} }, + { {-INF,-P12}, {U,U}, {U,U}, {U,U}, {U,U}, {-INF,P12}, {N,N} }, + { {-INF,-P12}, {U,U}, {-0.,-0.}, {-0.,0.}, {U,U}, {-INF,P12}, {N,N} }, + { {INF,-P12}, {U,U}, {0.,-0.}, {0.,0.}, {U,U}, {INF,P12}, {N,N} }, + { {INF,-P12}, {U,U}, {U,U}, {U,U}, {U,U}, {INF,P12}, {N,N} }, + { {INF,-P14}, {INF,-0.}, {INF,-0.}, {INF,0.}, {INF,0.}, {INF,P14}, {INF,N} }, + { {INF,N}, {N,N}, {N,-0.}, {N,0.}, {N,N}, {INF,N}, {N,N} } +}; /*[clinic input] cmath.asinh = cmath.acos @@ -318,7 +341,15 @@ cmath_atan_impl(PyObject *module, Py_complex z) } -static Py_complex atanh_special_values[7][7]; +static Py_complex atanh_special_values[7][7] = { + { {-0.,-P12}, {-0.,-P12}, {-0.,-P12}, {-0.,P12}, {-0.,P12}, {-0.,P12}, {-0.,N} }, + { {-0.,-P12}, {U,U}, {U,U}, {U,U}, {U,U}, {-0.,P12}, {N,N} }, + { {-0.,-P12}, {U,U}, {-0.,-0.}, {-0.,0.}, {U,U}, {-0.,P12}, {-0.,N} }, + { {0.,-P12}, {U,U}, {0.,-0.}, {0.,0.}, {U,U}, {0.,P12}, {0.,N} }, + { {0.,-P12}, {U,U}, {U,U}, {U,U}, {U,U}, {0.,P12}, {N,N} }, + { {0.,-P12}, {0.,-P12}, {0.,-P12}, {0.,P12}, {0.,P12}, {0.,P12}, {0.,N} }, + { {0.,-P12}, {N,N}, {N,N}, {N,N}, {N,N}, {0.,P12}, {N,N} } +}; /*[clinic input] cmath.atanh = cmath.acos @@ -391,7 +422,15 @@ cmath_cos_impl(PyObject *module, Py_complex z) /* cosh(infinity + i*y) needs to be dealt with specially */ -static Py_complex cosh_special_values[7][7]; +static Py_complex cosh_special_values[7][7] = { + { {INF,N}, {U,U}, {INF,0.}, {INF,-0.}, {U,U}, {INF,N}, {INF,N} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {N,0.}, {U,U}, {1.,0.}, {1.,-0.}, {U,U}, {N,0.}, {N,0.} }, + { {N,0.}, {U,U}, {1.,-0.}, {1.,0.}, {U,U}, {N,0.}, {N,0.} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {INF,N}, {U,U}, {INF,-0.}, {INF,0.}, {U,U}, {INF,N}, {INF,N} }, + { {N,N}, {N,N}, {N,0.}, {N,0.}, {N,N}, {N,N}, {N,N} } +}; /*[clinic input] cmath.cosh = cmath.acos @@ -453,7 +492,15 @@ cmath_cosh_impl(PyObject *module, Py_complex z) /* exp(infinity + i*y) and exp(-infinity + i*y) need special treatment for finite y */ -static Py_complex exp_special_values[7][7]; +static Py_complex exp_special_values[7][7] = { + { {0.,0.}, {U,U}, {0.,-0.}, {0.,0.}, {U,U}, {0.,0.}, {0.,0.} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {N,N}, {U,U}, {1.,-0.}, {1.,0.}, {U,U}, {N,N}, {N,N} }, + { {N,N}, {U,U}, {1.,-0.}, {1.,0.}, {U,U}, {N,N}, {N,N} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {INF,N}, {U,U}, {INF,-0.}, {INF,0.}, {U,U}, {INF,N}, {INF,N} }, + { {N,N}, {N,N}, {N,-0.}, {N,0.}, {N,N}, {N,N}, {N,N} } +}; /*[clinic input] cmath.exp = cmath.acos @@ -512,7 +559,15 @@ cmath_exp_impl(PyObject *module, Py_complex z) return r; } -static Py_complex log_special_values[7][7]; +static Py_complex log_special_values[7][7] = { + { {INF,-P34}, {INF,-P}, {INF,-P}, {INF,P}, {INF,P}, {INF,P34}, {INF,N} }, + { {INF,-P12}, {U,U}, {U,U}, {U,U}, {U,U}, {INF,P12}, {N,N} }, + { {INF,-P12}, {U,U}, {-INF,-P}, {-INF,P}, {U,U}, {INF,P12}, {N,N} }, + { {INF,-P12}, {U,U}, {-INF,-0.}, {-INF,0.}, {U,U}, {INF,P12}, {N,N} }, + { {INF,-P12}, {U,U}, {U,U}, {U,U}, {U,U}, {INF,P12}, {N,N} }, + { {INF,-P14}, {INF,-0.}, {INF,-0.}, {INF,0.}, {INF,0.}, {INF,P14}, {INF,N} }, + { {INF,N}, {N,N}, {N,N}, {N,N}, {N,N}, {INF,N}, {N,N} } +}; static Py_complex c_log(Py_complex z) @@ -628,7 +683,15 @@ cmath_sin_impl(PyObject *module, Py_complex z) /* sinh(infinity + i*y) needs to be dealt with specially */ -static Py_complex sinh_special_values[7][7]; +static Py_complex sinh_special_values[7][7] = { + { {INF,N}, {U,U}, {-INF,-0.}, {-INF,0.}, {U,U}, {INF,N}, {INF,N} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {0.,N}, {U,U}, {-0.,-0.}, {-0.,0.}, {U,U}, {0.,N}, {0.,N} }, + { {0.,N}, {U,U}, {0.,-0.}, {0.,0.}, {U,U}, {0.,N}, {0.,N} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {INF,N}, {U,U}, {INF,-0.}, {INF,0.}, {U,U}, {INF,N}, {INF,N} }, + { {N,N}, {N,N}, {N,-0.}, {N,0.}, {N,N}, {N,N}, {N,N} } +}; /*[clinic input] cmath.sinh = cmath.acos @@ -687,7 +750,15 @@ cmath_sinh_impl(PyObject *module, Py_complex z) } -static Py_complex sqrt_special_values[7][7]; +static Py_complex sqrt_special_values[7][7] = { + { {INF,-INF}, {0.,-INF}, {0.,-INF}, {0.,INF}, {0.,INF}, {INF,INF}, {N,INF} }, + { {INF,-INF}, {U,U}, {U,U}, {U,U}, {U,U}, {INF,INF}, {N,N} }, + { {INF,-INF}, {U,U}, {0.,-0.}, {0.,0.}, {U,U}, {INF,INF}, {N,N} }, + { {INF,-INF}, {U,U}, {0.,-0.}, {0.,0.}, {U,U}, {INF,INF}, {N,N} }, + { {INF,-INF}, {U,U}, {U,U}, {U,U}, {U,U}, {INF,INF}, {N,N} }, + { {INF,-INF}, {INF,-0.}, {INF,-0.}, {INF,0.}, {INF,0.}, {INF,INF}, {INF,N} }, + { {INF,-INF}, {N,N}, {N,N}, {N,N}, {N,N}, {INF,INF}, {N,N} } +}; /*[clinic input] cmath.sqrt = cmath.acos @@ -786,7 +857,15 @@ cmath_tan_impl(PyObject *module, Py_complex z) /* tanh(infinity + i*y) needs to be dealt with specially */ -static Py_complex tanh_special_values[7][7]; +static Py_complex tanh_special_values[7][7] = { + { {-1.,0.}, {U,U}, {-1.,-0.}, {-1.,0.}, {U,U}, {-1.,0.}, {-1.,0.} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {-0.0,N}, {U,U}, {-0.,-0.}, {-0.,0.}, {U,U}, {-0.0,N}, {-0.,N} }, + { {0.0,N}, {U,U}, {0.,-0.}, {0.,0.}, {U,U}, {0.0,N}, {0.,N} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {1.,0.}, {U,U}, {1.,-0.}, {1.,0.}, {U,U}, {1.,0.}, {1.,0.} }, + { {N,N}, {N,N}, {N,-0.}, {N,0.}, {N,N}, {N,N}, {N,N} } +}; /*[clinic input] cmath.tanh = cmath.acos @@ -969,7 +1048,15 @@ cmath_polar_impl(PyObject *module, Py_complex z) */ -static Py_complex rect_special_values[7][7]; +static Py_complex rect_special_values[7][7] = { + { {INF,N}, {U,U}, {-INF,0.}, {-INF,-0.}, {U,U}, {INF,N}, {INF,N} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {0.,0.}, {U,U}, {-0.,0.}, {-0.,-0.}, {U,U}, {0.,0.}, {0.,0.} }, + { {0.,0.}, {U,U}, {0.,-0.}, {0.,0.}, {U,U}, {0.,0.}, {0.,0.} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {INF,N}, {U,U}, {INF,-0.}, {INF,0.}, {U,U}, {INF,N}, {INF,N} }, + { {N,N}, {N,N}, {N,0.}, {N,0.}, {N,N}, {N,N}, {N,N} } +}; /*[clinic input] cmath.rect @@ -1202,120 +1289,6 @@ cmath_exec(PyObject *mod) return -1; } - /* initialize special value tables */ - -#define INIT_SPECIAL_VALUES(NAME, BODY) { Py_complex* p = (Py_complex*)NAME; BODY } -#define C(REAL, IMAG) p->real = REAL; p->imag = IMAG; ++p; - - INIT_SPECIAL_VALUES(acos_special_values, { - C(P34,INF) C(P,INF) C(P,INF) C(P,-INF) C(P,-INF) C(P34,-INF) C(N,INF) - C(P12,INF) C(U,U) C(U,U) C(U,U) C(U,U) C(P12,-INF) C(N,N) - C(P12,INF) C(U,U) C(P12,0.) C(P12,-0.) C(U,U) C(P12,-INF) C(P12,N) - C(P12,INF) C(U,U) C(P12,0.) C(P12,-0.) C(U,U) C(P12,-INF) C(P12,N) - C(P12,INF) C(U,U) C(U,U) C(U,U) C(U,U) C(P12,-INF) C(N,N) - C(P14,INF) C(0.,INF) C(0.,INF) C(0.,-INF) C(0.,-INF) C(P14,-INF) C(N,INF) - C(N,INF) C(N,N) C(N,N) C(N,N) C(N,N) C(N,-INF) C(N,N) - }) - - INIT_SPECIAL_VALUES(acosh_special_values, { - C(INF,-P34) C(INF,-P) C(INF,-P) C(INF,P) C(INF,P) C(INF,P34) C(INF,N) - C(INF,-P12) C(U,U) C(U,U) C(U,U) C(U,U) C(INF,P12) C(N,N) - C(INF,-P12) C(U,U) C(0.,-P12) C(0.,P12) C(U,U) C(INF,P12) C(N,P12) - C(INF,-P12) C(U,U) C(0.,-P12) C(0.,P12) C(U,U) C(INF,P12) C(N,P12) - C(INF,-P12) C(U,U) C(U,U) C(U,U) C(U,U) C(INF,P12) C(N,N) - C(INF,-P14) C(INF,-0.) C(INF,-0.) C(INF,0.) C(INF,0.) C(INF,P14) C(INF,N) - C(INF,N) C(N,N) C(N,N) C(N,N) C(N,N) C(INF,N) C(N,N) - }) - - INIT_SPECIAL_VALUES(asinh_special_values, { - C(-INF,-P14) C(-INF,-0.) C(-INF,-0.) C(-INF,0.) C(-INF,0.) C(-INF,P14) C(-INF,N) - C(-INF,-P12) C(U,U) C(U,U) C(U,U) C(U,U) C(-INF,P12) C(N,N) - C(-INF,-P12) C(U,U) C(-0.,-0.) C(-0.,0.) C(U,U) C(-INF,P12) C(N,N) - C(INF,-P12) C(U,U) C(0.,-0.) C(0.,0.) C(U,U) C(INF,P12) C(N,N) - C(INF,-P12) C(U,U) C(U,U) C(U,U) C(U,U) C(INF,P12) C(N,N) - C(INF,-P14) C(INF,-0.) C(INF,-0.) C(INF,0.) C(INF,0.) C(INF,P14) C(INF,N) - C(INF,N) C(N,N) C(N,-0.) C(N,0.) C(N,N) C(INF,N) C(N,N) - }) - - INIT_SPECIAL_VALUES(atanh_special_values, { - C(-0.,-P12) C(-0.,-P12) C(-0.,-P12) C(-0.,P12) C(-0.,P12) C(-0.,P12) C(-0.,N) - C(-0.,-P12) C(U,U) C(U,U) C(U,U) C(U,U) C(-0.,P12) C(N,N) - C(-0.,-P12) C(U,U) C(-0.,-0.) C(-0.,0.) C(U,U) C(-0.,P12) C(-0.,N) - C(0.,-P12) C(U,U) C(0.,-0.) C(0.,0.) C(U,U) C(0.,P12) C(0.,N) - C(0.,-P12) C(U,U) C(U,U) C(U,U) C(U,U) C(0.,P12) C(N,N) - C(0.,-P12) C(0.,-P12) C(0.,-P12) C(0.,P12) C(0.,P12) C(0.,P12) C(0.,N) - C(0.,-P12) C(N,N) C(N,N) C(N,N) C(N,N) C(0.,P12) C(N,N) - }) - - INIT_SPECIAL_VALUES(cosh_special_values, { - C(INF,N) C(U,U) C(INF,0.) C(INF,-0.) C(U,U) C(INF,N) C(INF,N) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(N,0.) C(U,U) C(1.,0.) C(1.,-0.) C(U,U) C(N,0.) C(N,0.) - C(N,0.) C(U,U) C(1.,-0.) C(1.,0.) C(U,U) C(N,0.) C(N,0.) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(INF,N) C(U,U) C(INF,-0.) C(INF,0.) C(U,U) C(INF,N) C(INF,N) - C(N,N) C(N,N) C(N,0.) C(N,0.) C(N,N) C(N,N) C(N,N) - }) - - INIT_SPECIAL_VALUES(exp_special_values, { - C(0.,0.) C(U,U) C(0.,-0.) C(0.,0.) C(U,U) C(0.,0.) C(0.,0.) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(N,N) C(U,U) C(1.,-0.) C(1.,0.) C(U,U) C(N,N) C(N,N) - C(N,N) C(U,U) C(1.,-0.) C(1.,0.) C(U,U) C(N,N) C(N,N) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(INF,N) C(U,U) C(INF,-0.) C(INF,0.) C(U,U) C(INF,N) C(INF,N) - C(N,N) C(N,N) C(N,-0.) C(N,0.) C(N,N) C(N,N) C(N,N) - }) - - INIT_SPECIAL_VALUES(log_special_values, { - C(INF,-P34) C(INF,-P) C(INF,-P) C(INF,P) C(INF,P) C(INF,P34) C(INF,N) - C(INF,-P12) C(U,U) C(U,U) C(U,U) C(U,U) C(INF,P12) C(N,N) - C(INF,-P12) C(U,U) C(-INF,-P) C(-INF,P) C(U,U) C(INF,P12) C(N,N) - C(INF,-P12) C(U,U) C(-INF,-0.) C(-INF,0.) C(U,U) C(INF,P12) C(N,N) - C(INF,-P12) C(U,U) C(U,U) C(U,U) C(U,U) C(INF,P12) C(N,N) - C(INF,-P14) C(INF,-0.) C(INF,-0.) C(INF,0.) C(INF,0.) C(INF,P14) C(INF,N) - C(INF,N) C(N,N) C(N,N) C(N,N) C(N,N) C(INF,N) C(N,N) - }) - - INIT_SPECIAL_VALUES(sinh_special_values, { - C(INF,N) C(U,U) C(-INF,-0.) C(-INF,0.) C(U,U) C(INF,N) C(INF,N) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(0.,N) C(U,U) C(-0.,-0.) C(-0.,0.) C(U,U) C(0.,N) C(0.,N) - C(0.,N) C(U,U) C(0.,-0.) C(0.,0.) C(U,U) C(0.,N) C(0.,N) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(INF,N) C(U,U) C(INF,-0.) C(INF,0.) C(U,U) C(INF,N) C(INF,N) - C(N,N) C(N,N) C(N,-0.) C(N,0.) C(N,N) C(N,N) C(N,N) - }) - - INIT_SPECIAL_VALUES(sqrt_special_values, { - C(INF,-INF) C(0.,-INF) C(0.,-INF) C(0.,INF) C(0.,INF) C(INF,INF) C(N,INF) - C(INF,-INF) C(U,U) C(U,U) C(U,U) C(U,U) C(INF,INF) C(N,N) - C(INF,-INF) C(U,U) C(0.,-0.) C(0.,0.) C(U,U) C(INF,INF) C(N,N) - C(INF,-INF) C(U,U) C(0.,-0.) C(0.,0.) C(U,U) C(INF,INF) C(N,N) - C(INF,-INF) C(U,U) C(U,U) C(U,U) C(U,U) C(INF,INF) C(N,N) - C(INF,-INF) C(INF,-0.) C(INF,-0.) C(INF,0.) C(INF,0.) C(INF,INF) C(INF,N) - C(INF,-INF) C(N,N) C(N,N) C(N,N) C(N,N) C(INF,INF) C(N,N) - }) - - INIT_SPECIAL_VALUES(tanh_special_values, { - C(-1.,0.) C(U,U) C(-1.,-0.) C(-1.,0.) C(U,U) C(-1.,0.) C(-1.,0.) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(-0.0,N) C(U,U) C(-0.,-0.) C(-0.,0.) C(U,U) C(-0.0,N) C(-0.,N) - C(0.0,N) C(U,U) C(0.,-0.) C(0.,0.) C(U,U) C(0.0,N) C(0.,N) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(1.,0.) C(U,U) C(1.,-0.) C(1.,0.) C(U,U) C(1.,0.) C(1.,0.) - C(N,N) C(N,N) C(N,-0.) C(N,0.) C(N,N) C(N,N) C(N,N) - }) - - INIT_SPECIAL_VALUES(rect_special_values, { - C(INF,N) C(U,U) C(-INF,0.) C(-INF,-0.) C(U,U) C(INF,N) C(INF,N) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(0.,0.) C(U,U) C(-0.,0.) C(-0.,-0.) C(U,U) C(0.,0.) C(0.,0.) - C(0.,0.) C(U,U) C(0.,-0.) C(0.,0.) C(U,U) C(0.,0.) C(0.,0.) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(INF,N) C(U,U) C(INF,-0.) C(INF,0.) C(U,U) C(INF,N) C(INF,N) - C(N,N) C(N,N) C(N,0.) C(N,0.) C(N,N) C(N,N) C(N,N) - }) return 0; } From b3bf2128989e550a7a02acbaa47389023b2c6bc9 Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Fri, 5 Dec 2025 04:28:08 +0800 Subject: [PATCH 258/638] gh-141976: Check stack bounds in JIT optimizer (GH-142201) --- Lib/test/test_generated_cases.py | 4 + ...-12-02-21-11-46.gh-issue-141976.yu7pDV.rst | 1 + Python/optimizer_analysis.c | 25 ++- Python/optimizer_cases.c.h | 173 ++++++++++++++++++ Tools/cases_generator/optimizer_generator.py | 2 +- Tools/cases_generator/stack.py | 13 +- 6 files changed, 212 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-12-02-21-11-46.gh-issue-141976.yu7pDV.rst diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index ac62e11c274fab..de0dbab480f5e5 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -2115,6 +2115,7 @@ def test_validate_uop_unused_input(self): """ output = """ case OP: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -2132,6 +2133,7 @@ def test_validate_uop_unused_input(self): """ output = """ case OP: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -2153,6 +2155,7 @@ def test_validate_uop_unused_output(self): case OP: { JitOptRef foo; foo = NULL; + CHECK_STACK_BOUNDS(1); stack_pointer[0] = foo; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2172,6 +2175,7 @@ def test_validate_uop_unused_output(self): """ output = """ case OP: { + CHECK_STACK_BOUNDS(1); stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-12-02-21-11-46.gh-issue-141976.yu7pDV.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-02-21-11-46.gh-issue-141976.yu7pDV.rst new file mode 100644 index 00000000000000..f77315b7c37d80 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-02-21-11-46.gh-issue-141976.yu7pDV.rst @@ -0,0 +1 @@ +Check against abstract stack overflow in the JIT optimizer. diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 685659ef7c46ef..51722556554609 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -144,10 +144,6 @@ incorrect_keys(PyObject *obj, uint32_t version) #define CURRENT_FRAME_IS_INIT_SHIM() (ctx->frame->code == ((PyCodeObject *)&_Py_InitCleanup)) -#define WITHIN_STACK_BOUNDS() \ - (CURRENT_FRAME_IS_INIT_SHIM() || (STACK_LEVEL() >= 0 && STACK_LEVEL() <= STACK_SIZE())) - - #define GETLOCAL(idx) ((ctx->frame->locals[idx])) #define REPLACE_OP(INST, OP, ARG, OPERAND) \ @@ -192,6 +188,27 @@ incorrect_keys(PyObject *obj, uint32_t version) #define JUMP_TO_LABEL(label) goto label; +static int +check_stack_bounds(JitOptContext *ctx, JitOptRef *stack_pointer, int offset, int opcode) +{ + int stack_level = (int)(stack_pointer + (offset) - ctx->frame->stack); + int should_check = !CURRENT_FRAME_IS_INIT_SHIM() || + (opcode == _RETURN_VALUE) || + (opcode == _RETURN_GENERATOR) || + (opcode == _YIELD_VALUE); + if (should_check && (stack_level < 0 || stack_level > STACK_SIZE())) { + ctx->contradiction = true; + ctx->done = true; + return 1; + } + return 0; +} + +#define CHECK_STACK_BOUNDS(offset) \ + if (check_stack_bounds(ctx, stack_pointer, offset, opcode)) { \ + break; \ + } \ + static int optimize_to_bool( _PyUOpInstruction *this_instr, diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 0c2d34d2e640a3..85bebed58677ed 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -33,6 +33,7 @@ if (sym_is_null(value)) { ctx->done = true; } + CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -42,6 +43,7 @@ case _LOAD_FAST: { JitOptRef value; value = GETLOCAL(oparg); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -51,6 +53,7 @@ case _LOAD_FAST_BORROW: { JitOptRef value; value = PyJitRef_Borrow(GETLOCAL(oparg)); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -62,6 +65,7 @@ value = GETLOCAL(oparg); JitOptRef temp = sym_new_null(ctx); GETLOCAL(oparg) = temp; + CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -74,6 +78,7 @@ PyObject *val = PyTuple_GET_ITEM(co->co_consts, oparg); REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); value = PyJitRef_Borrow(sym_new_const(ctx, val)); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -87,6 +92,7 @@ assert(_Py_IsImmortal(val)); REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); value = PyJitRef_Borrow(sym_new_const(ctx, val)); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -97,6 +103,7 @@ JitOptRef value; value = stack_pointer[-1]; GETLOCAL(oparg) = value; + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -120,36 +127,42 @@ else if (typ == &PyUnicode_Type) { REPLACE_OP(this_instr, _POP_TOP_UNICODE, 0, 0); } + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_TOP_NOP: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_TOP_INT: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_TOP_FLOAT: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_TOP_UNICODE: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_TWO: { + CHECK_STACK_BOUNDS(-2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -158,6 +171,7 @@ case _PUSH_NULL: { JitOptRef res; res = sym_new_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -165,12 +179,14 @@ } case _END_FOR: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_ITER: { + CHECK_STACK_BOUNDS(-2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -179,6 +195,7 @@ case _END_SEND: { JitOptRef val; val = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = val; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -504,12 +521,14 @@ REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } res = sym_new_compact_int(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -554,12 +573,14 @@ REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } res = sym_new_compact_int(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -604,12 +625,14 @@ REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } res = sym_new_compact_int(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -673,6 +696,7 @@ REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -682,6 +706,7 @@ if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -725,6 +750,7 @@ REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -734,6 +760,7 @@ if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -777,6 +804,7 @@ REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -786,6 +814,7 @@ if (PyJitRef_IsBorrowed(left) && PyJitRef_IsBorrowed(right)) { REPLACE_OP(this_instr, op_without_decref_inputs[opcode], oparg, 0); } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -795,6 +824,7 @@ case _BINARY_OP_MULTIPLY_FLOAT__NO_DECREF_INPUTS: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -804,6 +834,7 @@ case _BINARY_OP_ADD_FLOAT__NO_DECREF_INPUTS: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -813,6 +844,7 @@ case _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -856,12 +888,14 @@ REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } res = sym_new_type(ctx, &PyUnicode_Type); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -888,6 +922,7 @@ res = sym_new_type(ctx, &PyUnicode_Type); } GETLOCAL(this_instr->operand0) = res; + CHECK_STACK_BOUNDS(-2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -900,6 +935,7 @@ case _BINARY_OP_EXTEND: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -920,6 +956,7 @@ else { res = sym_new_not_null(ctx); } + CHECK_STACK_BOUNDS(-2); stack_pointer[-3] = res; stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -927,6 +964,7 @@ } case _STORE_SLICE: { + CHECK_STACK_BOUNDS(-4); stack_pointer += -4; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -935,6 +973,7 @@ case _BINARY_OP_SUBSCR_LIST_INT: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -944,6 +983,7 @@ case _BINARY_OP_SUBSCR_LIST_SLICE: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -953,6 +993,7 @@ case _BINARY_OP_SUBSCR_STR_INT: { JitOptRef res; res = sym_new_type(ctx, &PyUnicode_Type); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1002,6 +1043,7 @@ else { res = sym_new_not_null(ctx); } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1031,6 +1073,7 @@ case _BINARY_OP_SUBSCR_DICT: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1040,6 +1083,7 @@ case _BINARY_OP_SUBSCR_CHECK_FUNC: { JitOptRef getitem; getitem = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = getitem; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1050,6 +1094,7 @@ JitOptRef new_frame; new_frame = PyJitRef_NULL; ctx->done = true; + CHECK_STACK_BOUNDS(-2); stack_pointer[-3] = new_frame; stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1057,36 +1102,42 @@ } case _LIST_APPEND: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _SET_ADD: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_SUBSCR: { + CHECK_STACK_BOUNDS(-3); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_SUBSCR_LIST_INT: { + CHECK_STACK_BOUNDS(-3); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_SUBSCR_DICT: { + CHECK_STACK_BOUNDS(-3); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _DELETE_SUBSCR: { + CHECK_STACK_BOUNDS(-2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -1102,6 +1153,7 @@ case _CALL_INTRINSIC_2: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1113,6 +1165,7 @@ JitOptRef res; retval = stack_pointer[-1]; JitOptRef temp = PyJitRef_StripReferenceInfo(retval); + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); ctx->frame->stack_pointer = stack_pointer; @@ -1134,6 +1187,7 @@ } stack_pointer = ctx->frame->stack_pointer; res = temp; + CHECK_STACK_BOUNDS(1); stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1150,6 +1204,7 @@ case _GET_ANEXT: { JitOptRef awaitable; awaitable = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = awaitable; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1178,6 +1233,7 @@ JitOptRef value; retval = stack_pointer[-1]; JitOptRef temp = PyJitRef_StripReferenceInfo(retval); + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); ctx->frame->stack_pointer = stack_pointer; @@ -1193,6 +1249,7 @@ } stack_pointer = ctx->frame->stack_pointer; value = temp; + CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1200,6 +1257,7 @@ } case _POP_EXCEPT: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -1208,6 +1266,7 @@ case _LOAD_COMMON_CONSTANT: { JitOptRef value; value = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1217,6 +1276,7 @@ case _LOAD_BUILD_CLASS: { JitOptRef bc; bc = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = bc; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1224,6 +1284,7 @@ } case _STORE_NAME: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -1242,6 +1303,7 @@ for (int i = 0; i < oparg; i++) { values[i] = sym_new_unknown(ctx); } + CHECK_STACK_BOUNDS(-1 + oparg); stack_pointer += -1 + oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -1254,6 +1316,7 @@ seq = stack_pointer[-1]; val0 = sym_tuple_getitem(ctx, seq, 0); val1 = sym_tuple_getitem(ctx, seq, 1); + CHECK_STACK_BOUNDS(1); stack_pointer[-1] = val1; stack_pointer[0] = val0; stack_pointer += 1; @@ -1269,6 +1332,7 @@ for (int i = 0; i < oparg; i++) { values[i] = sym_tuple_getitem(ctx, seq, oparg - i - 1); } + CHECK_STACK_BOUNDS(-1 + oparg); stack_pointer += -1 + oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -1280,6 +1344,7 @@ for (int _i = oparg; --_i >= 0;) { values[_i] = sym_new_not_null(ctx); } + CHECK_STACK_BOUNDS(-1 + oparg); stack_pointer += -1 + oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -1295,24 +1360,28 @@ for (int i = 0; i < totalargs; i++) { values[i] = sym_new_unknown(ctx); } + CHECK_STACK_BOUNDS((oparg & 0xFF) + (oparg >> 8)); stack_pointer += (oparg & 0xFF) + (oparg >> 8); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_ATTR: { + CHECK_STACK_BOUNDS(-2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _DELETE_ATTR: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_GLOBAL: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -1325,6 +1394,7 @@ case _LOAD_LOCALS: { JitOptRef locals; locals = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = locals; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1336,6 +1406,7 @@ case _LOAD_NAME: { JitOptRef v; v = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = v; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1346,6 +1417,7 @@ JitOptRef *res; res = &stack_pointer[0]; res[0] = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -1361,6 +1433,7 @@ else { REPLACE_OP(this_instr, _NOP, 0, 0); } + CHECK_STACK_BOUNDS((oparg & 1)); stack_pointer += (oparg & 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -1426,6 +1499,7 @@ else { res = sym_new_const(ctx, cnst); } + CHECK_STACK_BOUNDS(1); stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1462,6 +1536,7 @@ else { res = sym_new_const(ctx, cnst); } + CHECK_STACK_BOUNDS(1); stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1490,6 +1565,7 @@ case _LOAD_DEREF: { JitOptRef value; value = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1497,6 +1573,7 @@ } case _STORE_DEREF: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -1509,6 +1586,7 @@ case _BUILD_STRING: { JitOptRef str; str = sym_new_type(ctx, &PyUnicode_Type); + CHECK_STACK_BOUNDS(1 - oparg); stack_pointer[-oparg] = str; stack_pointer += 1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1518,6 +1596,7 @@ case _BUILD_INTERPOLATION: { JitOptRef interpolation; interpolation = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1 - (oparg & 1)); stack_pointer[-2 - (oparg & 1)] = interpolation; stack_pointer += -1 - (oparg & 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1527,6 +1606,7 @@ case _BUILD_TEMPLATE: { JitOptRef template; template = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = template; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1538,6 +1618,7 @@ JitOptRef tup; values = &stack_pointer[-oparg]; tup = sym_new_tuple(ctx, oparg, values); + CHECK_STACK_BOUNDS(1 - oparg); stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1547,6 +1628,7 @@ case _BUILD_LIST: { JitOptRef list; list = sym_new_type(ctx, &PyList_Type); + CHECK_STACK_BOUNDS(1 - oparg); stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1554,12 +1636,14 @@ } case _LIST_EXTEND: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _SET_UPDATE: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -1568,6 +1652,7 @@ case _BUILD_SET: { JitOptRef set; set = sym_new_type(ctx, &PySet_Type); + CHECK_STACK_BOUNDS(1 - oparg); stack_pointer[-oparg] = set; stack_pointer += 1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1577,6 +1662,7 @@ case _BUILD_MAP: { JitOptRef map; map = sym_new_type(ctx, &PyDict_Type); + CHECK_STACK_BOUNDS(1 - oparg*2); stack_pointer[-oparg*2] = map; stack_pointer += 1 - oparg*2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1588,18 +1674,21 @@ } case _DICT_UPDATE: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _DICT_MERGE: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _MAP_ADD: { + CHECK_STACK_BOUNDS(-2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -1608,6 +1697,7 @@ case _LOAD_SUPER_ATTR_ATTR: { JitOptRef attr_st; attr_st = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-2); stack_pointer[-3] = attr_st; stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1619,6 +1709,7 @@ JitOptRef self_or_null; attr = sym_new_not_null(ctx); self_or_null = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-3] = attr; stack_pointer[-2] = self_or_null; stack_pointer += -1; @@ -1638,6 +1729,7 @@ if (oparg & 1) { self_or_null[0] = sym_new_unknown(ctx); } + CHECK_STACK_BOUNDS((oparg&1)); stack_pointer += (oparg&1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -1780,18 +1872,21 @@ } case _STORE_ATTR_INSTANCE_VALUE: { + CHECK_STACK_BOUNDS(-2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_ATTR_WITH_HINT: { + CHECK_STACK_BOUNDS(-2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _STORE_ATTR_SLOT: { + CHECK_STACK_BOUNDS(-2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -1840,6 +1935,7 @@ REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1851,6 +1947,7 @@ else { res = _Py_uop_sym_new_not_null(ctx); } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1891,12 +1988,14 @@ REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } res = sym_new_type(ctx, &PyBool_Type); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1941,12 +2040,14 @@ REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } res = sym_new_type(ctx, &PyBool_Type); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1989,12 +2090,14 @@ REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } res = sym_new_type(ctx, &PyBool_Type); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2004,6 +2107,7 @@ case _IS_OP: { JitOptRef b; b = sym_new_type(ctx, &PyBool_Type); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = b; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2042,12 +2146,14 @@ REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = b; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } b = sym_new_type(ctx, &PyBool_Type); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = b; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2068,6 +2174,7 @@ case _CONTAINS_OP_SET: { JitOptRef b; b = sym_new_type(ctx, &PyBool_Type); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = b; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2077,6 +2184,7 @@ case _CONTAINS_OP_DICT: { JitOptRef b; b = sym_new_type(ctx, &PyBool_Type); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = b; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2103,6 +2211,7 @@ case _IMPORT_NAME: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2112,6 +2221,7 @@ case _IMPORT_FROM: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2149,12 +2259,14 @@ REPLACE_OP(this_instr, _LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)temp); } len = sym_new_const(ctx, temp); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = len; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); Py_DECREF(temp); stack_pointer += -1; } + CHECK_STACK_BOUNDS(1); stack_pointer[0] = len; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2164,6 +2276,7 @@ case _MATCH_CLASS: { JitOptRef attrs; attrs = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-2); stack_pointer[-3] = attrs; stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2173,6 +2286,7 @@ case _MATCH_MAPPING: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2182,6 +2296,7 @@ case _MATCH_SEQUENCE: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2191,6 +2306,7 @@ case _MATCH_KEYS: { JitOptRef values_or_none; values_or_none = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = values_or_none; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2210,6 +2326,7 @@ iter = sym_new_not_null(ctx); index_or_null = sym_new_unknown(ctx); } + CHECK_STACK_BOUNDS(1); stack_pointer[-1] = iter; stack_pointer[0] = index_or_null; stack_pointer += 1; @@ -2229,6 +2346,7 @@ case _FOR_ITER_TIER_TWO: { JitOptRef next; next = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = next; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2252,6 +2370,7 @@ case _ITER_NEXT_LIST_TIER_TWO: { JitOptRef next; next = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = next; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2277,6 +2396,7 @@ case _ITER_NEXT_TUPLE: { JitOptRef next; next = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = next; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2296,6 +2416,7 @@ case _ITER_NEXT_RANGE: { JitOptRef next; next = sym_new_type(ctx, &PyLong_Type); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = next; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2306,6 +2427,7 @@ JitOptRef gen_frame; gen_frame = PyJitRef_NULL; ctx->done = true; + CHECK_STACK_BOUNDS(1); stack_pointer[0] = gen_frame; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2319,6 +2441,7 @@ method_and_self = &stack_pointer[-1]; method_and_self[0] = sym_new_null(ctx); method_and_self[1] = self; + CHECK_STACK_BOUNDS(1); stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -2335,6 +2458,7 @@ case _WITH_EXCEPT_START: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2346,6 +2470,7 @@ JitOptRef new_exc; prev_exc = sym_new_not_null(ctx); new_exc = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[-1] = prev_exc; stack_pointer[0] = new_exc; stack_pointer += 1; @@ -2374,6 +2499,7 @@ _LOAD_CONST_UNDER_INLINE_BORROW, _LOAD_CONST_UNDER_INLINE); self = owner; + CHECK_STACK_BOUNDS(1); stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; @@ -2394,6 +2520,7 @@ _LOAD_CONST_UNDER_INLINE_BORROW, _LOAD_CONST_UNDER_INLINE); self = owner; + CHECK_STACK_BOUNDS(1); stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; @@ -2448,6 +2575,7 @@ _LOAD_CONST_UNDER_INLINE_BORROW, _LOAD_CONST_UNDER_INLINE); self = owner; + CHECK_STACK_BOUNDS(1); stack_pointer[-1] = attr; stack_pointer[0] = self; stack_pointer += 1; @@ -2483,6 +2611,7 @@ break; } new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); + CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2531,6 +2660,7 @@ case _CALL_NON_PY_GENERAL: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2616,6 +2746,7 @@ } else { new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); } + CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = new_frame; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2625,6 +2756,7 @@ case _PUSH_FRAME: { JitOptRef new_frame; new_frame = stack_pointer[-1]; + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (!CURRENT_FRAME_IS_INIT_SHIM()) { @@ -2703,6 +2835,7 @@ else { res = sym_new_not_null(ctx); } + CHECK_STACK_BOUNDS(-2); stack_pointer[-3] = res; stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2729,6 +2862,7 @@ else { res = sym_new_type(ctx, &PyUnicode_Type); } + CHECK_STACK_BOUNDS(-2); stack_pointer[-3] = res; stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2755,6 +2889,7 @@ else { res = sym_new_type(ctx, &PyTuple_Type); } + CHECK_STACK_BOUNDS(-2); stack_pointer[-3] = res; stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2797,6 +2932,7 @@ assert((this_instr + 1)->opcode == _PUSH_FRAME); PyCodeObject *co = get_code_with_logging((this_instr + 1)); init_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, args-1, oparg+1)); + CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = init_frame; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2804,6 +2940,7 @@ } case _EXIT_INIT_CHECK: { + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -2812,6 +2949,7 @@ case _CALL_BUILTIN_CLASS: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2821,6 +2959,7 @@ case _CALL_BUILTIN_O: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2830,6 +2969,7 @@ case _CALL_BUILTIN_FAST: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2839,6 +2979,7 @@ case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2872,12 +3013,14 @@ 0, (uintptr_t)temp); } res = sym_new_const(ctx, temp); + CHECK_STACK_BOUNDS(-2); stack_pointer[-3] = res; stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); Py_DECREF(temp); stack_pointer += 2; } + CHECK_STACK_BOUNDS(-2); stack_pointer[-3] = res; stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2912,6 +3055,7 @@ sym_set_const(res, out); REPLACE_OP(this_instr, _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)out); } + CHECK_STACK_BOUNDS(-3); stack_pointer[-4] = res; stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2930,6 +3074,7 @@ } case _CALL_LIST_APPEND: { + CHECK_STACK_BOUNDS(-3); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -2938,6 +3083,7 @@ case _CALL_METHOD_DESCRIPTOR_O: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2947,6 +3093,7 @@ case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2956,6 +3103,7 @@ case _CALL_METHOD_DESCRIPTOR_NOARGS: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2965,6 +3113,7 @@ case _CALL_METHOD_DESCRIPTOR_FAST: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1 - oparg); stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -2988,6 +3137,7 @@ break; } new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); + CHECK_STACK_BOUNDS(-2 - oparg); stack_pointer[-3 - oparg] = new_frame; stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3013,6 +3163,7 @@ case _CALL_KW_NON_PY: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-2 - oparg); stack_pointer[-3 - oparg] = res; stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3035,6 +3186,7 @@ case _SET_FUNCTION_ATTRIBUTE: { JitOptRef func_out; func_out = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = func_out; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3056,6 +3208,7 @@ } stack_pointer = ctx->frame->stack_pointer; res = sym_new_unknown(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = res; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3065,6 +3218,7 @@ case _BUILD_SLICE: { JitOptRef slice; slice = sym_new_type(ctx, &PySlice_Type); + CHECK_STACK_BOUNDS(1 - oparg); stack_pointer[-oparg] = slice; stack_pointer += 1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3088,6 +3242,7 @@ case _FORMAT_WITH_SPEC: { JitOptRef res; res = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3100,6 +3255,7 @@ bottom = stack_pointer[-1 - (oparg-1)]; assert(oparg > 0); top = bottom; + CHECK_STACK_BOUNDS(1); stack_pointer[0] = top; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3139,6 +3295,7 @@ REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); } } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3177,6 +3334,7 @@ else { res = sym_new_type(ctx, &PyFloat_Type); } + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3224,6 +3382,7 @@ eliminate_pop_guard(this_instr, value != Py_True); } sym_set_const(flag, Py_True); + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -3238,6 +3397,7 @@ eliminate_pop_guard(this_instr, value != Py_False); } sym_set_const(flag, Py_False); + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -3256,6 +3416,7 @@ eliminate_pop_guard(this_instr, true); } sym_set_const(val, Py_None); + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -3273,6 +3434,7 @@ assert(!sym_matches_type(val, &_PyNone_Type)); eliminate_pop_guard(this_instr, false); } + CHECK_STACK_BOUNDS(-1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -3317,6 +3479,7 @@ JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; value = sym_new_const(ctx, ptr); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3335,6 +3498,7 @@ JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3342,18 +3506,21 @@ } case _POP_CALL: { + CHECK_STACK_BOUNDS(-2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_CALL_ONE: { + CHECK_STACK_BOUNDS(-3); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } case _POP_CALL_TWO: { + CHECK_STACK_BOUNDS(-4); stack_pointer += -4; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -3370,6 +3537,7 @@ case _POP_TWO_LOAD_CONST_INLINE_BORROW: { JitOptRef value; value = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = value; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3380,6 +3548,7 @@ JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + CHECK_STACK_BOUNDS(-1); stack_pointer[-2] = value; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3390,6 +3559,7 @@ JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + CHECK_STACK_BOUNDS(-2); stack_pointer[-3] = value; stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3400,6 +3570,7 @@ JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + CHECK_STACK_BOUNDS(-3); stack_pointer[-4] = value; stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3411,6 +3582,7 @@ JitOptRef new; value = sym_new_not_null(ctx); new = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[-1] = value; stack_pointer[0] = new; stack_pointer += 1; @@ -3423,6 +3595,7 @@ JitOptRef new; value = sym_new_not_null(ctx); new = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); stack_pointer[-1] = value; stack_pointer[0] = new; stack_pointer += 1; diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index 41df073cf6df23..ab0a90e234b124 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -445,7 +445,7 @@ def generate_abstract_interpreter( declare_variables(override, out, skip_inputs=False) else: declare_variables(uop, out, skip_inputs=True) - stack = Stack() + stack = Stack(check_stack_bounds=True) write_uop(override, uop, out, stack, debug, skip_inputs=(override is None)) out.start_line() out.emit("break;\n") diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index 6519e8e4f3ed36..53499558aed876 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -216,11 +216,12 @@ def array_or_scalar(var: StackItem | Local) -> str: return "array" if var.is_array() else "scalar" class Stack: - def __init__(self) -> None: + def __init__(self, check_stack_bounds: bool = False) -> None: self.base_offset = PointerOffset.zero() self.physical_sp = PointerOffset.zero() self.logical_sp = PointerOffset.zero() self.variables: list[Local] = [] + self.check_stack_bounds = check_stack_bounds def drop(self, var: StackItem, check_liveness: bool) -> None: self.logical_sp = self.logical_sp.pop(var) @@ -316,8 +317,17 @@ def save_variables(self, out: CWriter) -> None: self._print(out) var_offset = var_offset.push(var.item) + def stack_bound_check(self, out: CWriter) -> None: + if not self.check_stack_bounds: + return + if self.physical_sp != self.logical_sp: + diff = self.logical_sp - self.physical_sp + out.start_line() + out.emit(f"CHECK_STACK_BOUNDS({diff});\n") + def flush(self, out: CWriter) -> None: self._print(out) + self.stack_bound_check(out) self.save_variables(out) self._save_physical_sp(out) out.start_line() @@ -347,6 +357,7 @@ def copy(self) -> "Stack": other.physical_sp = self.physical_sp other.logical_sp = self.logical_sp other.variables = [var.copy() for var in self.variables] + other.check_stack_bounds = self.check_stack_bounds return other def __eq__(self, other: object) -> bool: From 128d31637e8bf23f086d2a09081525adeeb1f65a Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Thu, 4 Dec 2025 20:15:04 -0600 Subject: [PATCH 259/638] gh-141926: Do not unset `RUNSHARED` when cross-compiling (#141958) --- .../next/Build/2025-11-25-13-17-47.gh-issue-141926.KmuM2h.rst | 4 ++++ configure | 4 ---- configure.ac | 4 ---- 3 files changed, 4 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2025-11-25-13-17-47.gh-issue-141926.KmuM2h.rst diff --git a/Misc/NEWS.d/next/Build/2025-11-25-13-17-47.gh-issue-141926.KmuM2h.rst b/Misc/NEWS.d/next/Build/2025-11-25-13-17-47.gh-issue-141926.KmuM2h.rst new file mode 100644 index 00000000000000..dab79ba5cf949d --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-11-25-13-17-47.gh-issue-141926.KmuM2h.rst @@ -0,0 +1,4 @@ +``RUNSHARED`` is no longer cleared when cross-compiling. Previously, +``RUNSHARED`` was cleared when cross-compiling, which breaks PGO when using +``--enabled-shared`` on systems where the cross-compiled CPython is otherwise +executable (e.g., via transparent emulation). diff --git a/configure b/configure index 620878bb181378..1561f7f4134ac2 100755 --- a/configure +++ b/configure @@ -7808,10 +7808,6 @@ fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $LDLIBRARY" >&5 printf "%s\n" "$LDLIBRARY" >&6; } -if test "$cross_compiling" = yes; then - RUNSHARED= -fi - # HOSTRUNNER - Program to run CPython for the host platform { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking HOSTRUNNER" >&5 printf %s "checking HOSTRUNNER... " >&6; } diff --git a/configure.ac b/configure.ac index 8ef479fe32036c..f2a7319d22d24b 100644 --- a/configure.ac +++ b/configure.ac @@ -1639,10 +1639,6 @@ else # shared is disabled fi AC_MSG_RESULT([$LDLIBRARY]) -if test "$cross_compiling" = yes; then - RUNSHARED= -fi - # HOSTRUNNER - Program to run CPython for the host platform AC_MSG_CHECKING([HOSTRUNNER]) if test -z "$HOSTRUNNER" From 53ec7c8fc07eb6958869638a0cad70c52ad6fcf5 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 4 Dec 2025 20:04:42 -0800 Subject: [PATCH 260/638] gh-142214: Fix two regressions in dataclasses (#142223) --- Lib/dataclasses.py | 14 +++++++--- Lib/test/test_dataclasses/__init__.py | 28 +++++++++++++++++++ ...-12-03-06-12-39.gh-issue-142214.appYNZ.rst | 12 ++++++++ 3 files changed, 50 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-12-03-06-12-39.gh-issue-142214.appYNZ.rst diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 3ccb72469286eb..730ced7299865e 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -550,7 +550,12 @@ def __annotate__(format, /): new_annotations = {} for k in annotation_fields: - new_annotations[k] = cls_annotations[k] + # gh-142214: The annotation may be missing in unusual dynamic cases. + # If so, just skip it. + try: + new_annotations[k] = cls_annotations[k] + except KeyError: + pass if return_type is not MISSING: if format == Format.STRING: @@ -1399,9 +1404,10 @@ def _add_slots(cls, is_frozen, weakref_slot, defined_fields): f.type = ann # Fix the class reference in the __annotate__ method - init_annotate = newcls.__init__.__annotate__ - if getattr(init_annotate, "__generated_by_dataclasses__", False): - _update_func_cell_for__class__(init_annotate, cls, newcls) + init = newcls.__init__ + if init_annotate := getattr(init, "__annotate__", None): + if getattr(init_annotate, "__generated_by_dataclasses__", False): + _update_func_cell_for__class__(init_annotate, cls, newcls) return newcls diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index 513dd78c4381b4..3b335429b98500 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -927,6 +927,20 @@ class C: validate_class(C) + def test_incomplete_annotations(self): + # gh-142214 + @dataclass + class C: + "doc" # needed because otherwise we fetch the annotations at the wrong time + x: int + + C.__annotate__ = lambda _: {} + + self.assertEqual( + annotationlib.get_annotations(C.__init__), + {"return": None} + ) + def test_missing_default(self): # Test that MISSING works the same as a default not being # specified. @@ -2578,6 +2592,20 @@ def __init__(self, x: int) -> None: self.assertFalse(hasattr(E.__init__.__annotate__, "__generated_by_dataclasses__")) + def test_slots_true_init_false(self): + # Test that slots=True and init=False work together and + # that __annotate__ is not added to __init__. + + @dataclass(slots=True, init=False) + class F: + x: int + + f = F() + f.x = 10 + self.assertEqual(f.x, 10) + + self.assertFalse(hasattr(F.__init__, "__annotate__")) + def test_init_false_forwardref(self): # Test forward references in fields not required for __init__ annotations. diff --git a/Misc/NEWS.d/next/Library/2025-12-03-06-12-39.gh-issue-142214.appYNZ.rst b/Misc/NEWS.d/next/Library/2025-12-03-06-12-39.gh-issue-142214.appYNZ.rst new file mode 100644 index 00000000000000..b87430ec1a3d65 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-03-06-12-39.gh-issue-142214.appYNZ.rst @@ -0,0 +1,12 @@ +Fix two regressions in :mod:`dataclasses` in Python 3.14.1 related to +annotations. + +* An exception is no longer raised if ``slots=True`` is used and the + ``__init__`` method does not have an ``__annotate__`` attribute + (likely because ``init=False`` was used). + +* An exception is no longer raised if annotations are requested on the + ``__init__`` method and one of the fields is not present in the class + annotations. This can occur in certain dynamic scenarios. + +Patch by Jelle Zijlstra. From 4238a975d78a0cc8f1751cfc63b3030b94b46aa8 Mon Sep 17 00:00:00 2001 From: Sanyam Khurana <8039608+CuriousLearner@users.noreply.github.com> Date: Fri, 5 Dec 2025 07:18:54 -0500 Subject: [PATCH 261/638] gh-48752: Add readline.get_pre_input_hook() function (#141586) Add readline.get_pre_input_hook() to retrieve the current pre-input hook. This allows applications to save and restore the hook without overwriting user settings. --- Doc/library/readline.rst | 9 ++++++ Lib/test/test_readline.py | 18 ++++++++++++ ...5-11-15-11-10-16.gh-issue-48752.aB3xYz.rst | 3 ++ Modules/clinic/readline.c.h | 28 ++++++++++++++++++- Modules/readline.c | 21 ++++++++++++++ 5 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-15-11-10-16.gh-issue-48752.aB3xYz.rst diff --git a/Doc/library/readline.rst b/Doc/library/readline.rst index 75db832c546b64..780cc77340366a 100644 --- a/Doc/library/readline.rst +++ b/Doc/library/readline.rst @@ -246,6 +246,15 @@ Startup hooks if Python was compiled for a version of the library that supports it. +.. function:: get_pre_input_hook() + + Get the current pre-input hook function, or ``None`` if no pre-input hook + function has been set. This function only exists if Python was compiled + for a version of the library that supports it. + + .. versionadded:: next + + .. _readline-completion: Completion diff --git a/Lib/test/test_readline.py b/Lib/test/test_readline.py index 45192fe508270d..3982686dd10aec 100644 --- a/Lib/test/test_readline.py +++ b/Lib/test/test_readline.py @@ -413,6 +413,24 @@ def test_write_read_limited_history(self): # So, we've only tested that the read did not fail. # See TestHistoryManipulation for the full test. + @unittest.skipUnless(hasattr(readline, "get_pre_input_hook"), + "get_pre_input_hook not available") + def test_get_pre_input_hook(self): + # Save and restore the original hook to avoid side effects + original_hook = readline.get_pre_input_hook() + self.addCleanup(readline.set_pre_input_hook, original_hook) + + # Test that get_pre_input_hook returns None when no hook is set + readline.set_pre_input_hook(None) + self.assertIsNone(readline.get_pre_input_hook()) + + # Set a hook and verify we can retrieve it + def my_hook(): + pass + + readline.set_pre_input_hook(my_hook) + self.assertIs(readline.get_pre_input_hook(), my_hook) + @unittest.skipUnless(support.Py_GIL_DISABLED, 'these tests can only possibly fail with GIL disabled') class FreeThreadingTest(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2025-11-15-11-10-16.gh-issue-48752.aB3xYz.rst b/Misc/NEWS.d/next/Library/2025-11-15-11-10-16.gh-issue-48752.aB3xYz.rst new file mode 100644 index 00000000000000..37b91196658589 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-15-11-10-16.gh-issue-48752.aB3xYz.rst @@ -0,0 +1,3 @@ +Add :func:`readline.get_pre_input_hook` function to retrieve the current +pre-input hook. This allows applications to save and restore the hook +without overwriting user settings. Patch by Sanyam Khurana. diff --git a/Modules/clinic/readline.c.h b/Modules/clinic/readline.c.h index 696475f7d00f5b..dc9381e4b976ac 100644 --- a/Modules/clinic/readline.c.h +++ b/Modules/clinic/readline.c.h @@ -349,6 +349,28 @@ readline_set_pre_input_hook(PyObject *module, PyObject *const *args, Py_ssize_t #endif /* defined(HAVE_RL_PRE_INPUT_HOOK) */ +#if defined(HAVE_RL_PRE_INPUT_HOOK) + +PyDoc_STRVAR(readline_get_pre_input_hook__doc__, +"get_pre_input_hook($module, /)\n" +"--\n" +"\n" +"Get the current pre-input hook function."); + +#define READLINE_GET_PRE_INPUT_HOOK_METHODDEF \ + {"get_pre_input_hook", (PyCFunction)readline_get_pre_input_hook, METH_NOARGS, readline_get_pre_input_hook__doc__}, + +static PyObject * +readline_get_pre_input_hook_impl(PyObject *module); + +static PyObject * +readline_get_pre_input_hook(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return readline_get_pre_input_hook_impl(module); +} + +#endif /* defined(HAVE_RL_PRE_INPUT_HOOK) */ + PyDoc_STRVAR(readline_get_completion_type__doc__, "get_completion_type($module, /)\n" "--\n" @@ -794,7 +816,11 @@ readline_redisplay(PyObject *module, PyObject *Py_UNUSED(ignored)) #define READLINE_SET_PRE_INPUT_HOOK_METHODDEF #endif /* !defined(READLINE_SET_PRE_INPUT_HOOK_METHODDEF) */ +#ifndef READLINE_GET_PRE_INPUT_HOOK_METHODDEF + #define READLINE_GET_PRE_INPUT_HOOK_METHODDEF +#endif /* !defined(READLINE_GET_PRE_INPUT_HOOK_METHODDEF) */ + #ifndef READLINE_CLEAR_HISTORY_METHODDEF #define READLINE_CLEAR_HISTORY_METHODDEF #endif /* !defined(READLINE_CLEAR_HISTORY_METHODDEF) */ -/*[clinic end generated code: output=88d9812b6caa2102 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=4bd95070973cd0e2 input=a9049054013a1b77]*/ diff --git a/Modules/readline.c b/Modules/readline.c index e89755b0cb4b2a..cc84eb6229e66d 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -572,6 +572,26 @@ readline_set_pre_input_hook_impl(PyObject *module, PyObject *function) return set_hook("pre_input_hook", &state->pre_input_hook, function); } + +/* Get pre-input hook */ + +/*[clinic input] +readline.get_pre_input_hook + +Get the current pre-input hook function. +[clinic start generated code]*/ + +static PyObject * +readline_get_pre_input_hook_impl(PyObject *module) +/*[clinic end generated code: output=ad56b77a8e8981ca input=fb1e1b1fbd94e4e5]*/ +{ + readlinestate *state = get_readline_state(module); + if (state->pre_input_hook == NULL) { + Py_RETURN_NONE; + } + return Py_NewRef(state->pre_input_hook); +} + #endif @@ -1074,6 +1094,7 @@ static struct PyMethodDef readline_methods[] = READLINE_SET_STARTUP_HOOK_METHODDEF #ifdef HAVE_RL_PRE_INPUT_HOOK READLINE_SET_PRE_INPUT_HOOK_METHODDEF + READLINE_GET_PRE_INPUT_HOOK_METHODDEF #endif #ifdef HAVE_RL_COMPLETION_APPEND_CHARACTER READLINE_CLEAR_HISTORY_METHODDEF From cac4b04973ea4cee80b775782453cddcd694635d Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Fri, 5 Dec 2025 13:10:51 +0000 Subject: [PATCH 262/638] Fix disk space issues in Android CI (#142289) --- Android/android.py | 39 +++++++++++++++++++--------- Android/testbed/app/build.gradle.kts | 4 +-- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/Android/android.py b/Android/android.py index 25bb4ca70b581f..d1a10be776ed16 100755 --- a/Android/android.py +++ b/Android/android.py @@ -29,6 +29,7 @@ ANDROID_DIR.name == "Android" and (PYTHON_DIR / "pyconfig.h.in").exists() ) +ENV_SCRIPT = ANDROID_DIR / "android-env.sh" TESTBED_DIR = ANDROID_DIR / "testbed" CROSS_BUILD_DIR = PYTHON_DIR / "cross-build" @@ -129,12 +130,11 @@ def android_env(host): sysconfig_filename = next(sysconfig_files).name host = re.fullmatch(r"_sysconfigdata__android_(.+).py", sysconfig_filename)[1] - env_script = ANDROID_DIR / "android-env.sh" env_output = subprocess.run( f"set -eu; " f"HOST={host}; " f"PREFIX={prefix}; " - f". {env_script}; " + f". {ENV_SCRIPT}; " f"export", check=True, shell=True, capture_output=True, encoding='utf-8', ).stdout @@ -151,7 +151,7 @@ def android_env(host): env[key] = value if not env: - raise ValueError(f"Found no variables in {env_script.name} output:\n" + raise ValueError(f"Found no variables in {ENV_SCRIPT.name} output:\n" + env_output) return env @@ -281,15 +281,30 @@ def clean_all(context): def setup_ci(): - # https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/ - if "GITHUB_ACTIONS" in os.environ and platform.system() == "Linux": - run( - ["sudo", "tee", "/etc/udev/rules.d/99-kvm4all.rules"], - input='KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"\n', - text=True, - ) - run(["sudo", "udevadm", "control", "--reload-rules"]) - run(["sudo", "udevadm", "trigger", "--name-match=kvm"]) + if "GITHUB_ACTIONS" in os.environ: + # Enable emulator hardware acceleration + # (https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/). + if platform.system() == "Linux": + run( + ["sudo", "tee", "/etc/udev/rules.d/99-kvm4all.rules"], + input='KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"\n', + text=True, + ) + run(["sudo", "udevadm", "control", "--reload-rules"]) + run(["sudo", "udevadm", "trigger", "--name-match=kvm"]) + + # Free up disk space by deleting unused versions of the NDK + # (https://github.com/freakboy3742/pyspamsum/pull/108). + for line in ENV_SCRIPT.read_text().splitlines(): + if match := re.fullmatch(r"ndk_version=(.+)", line): + ndk_version = match[1] + break + else: + raise ValueError(f"Failed to find NDK version in {ENV_SCRIPT.name}") + + for item in (android_home / "ndk").iterdir(): + if item.name[0].isdigit() and item.name != ndk_version: + delete_glob(item) def setup_sdk(): diff --git a/Android/testbed/app/build.gradle.kts b/Android/testbed/app/build.gradle.kts index 4de628a279ca3f..14d43d8c4d5c42 100644 --- a/Android/testbed/app/build.gradle.kts +++ b/Android/testbed/app/build.gradle.kts @@ -79,7 +79,7 @@ android { val androidEnvFile = file("../../android-env.sh").absoluteFile namespace = "org.python.testbed" - compileSdk = 34 + compileSdk = 35 defaultConfig { applicationId = "org.python.testbed" @@ -92,7 +92,7 @@ android { } throw GradleException("Failed to find API level in $androidEnvFile") } - targetSdk = 34 + targetSdk = 35 versionCode = 1 versionName = "1.0" From 1d8f3ed2eba762e60a02ff87e782a5c7dcd0e77c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 5 Dec 2025 16:22:38 +0200 Subject: [PATCH 263/638] gh-101100: Fix references to the set methods (GH-141857) --- Doc/c-api/set.rst | 2 +- Doc/library/stdtypes.rst | 210 +++++++++++++++++++----------------- Doc/reference/datamodel.rst | 2 +- Doc/whatsnew/2.3.rst | 4 +- 4 files changed, 113 insertions(+), 105 deletions(-) diff --git a/Doc/c-api/set.rst b/Doc/c-api/set.rst index cba823aa027bd6..09c0fb6b9c5f23 100644 --- a/Doc/c-api/set.rst +++ b/Doc/c-api/set.rst @@ -147,7 +147,7 @@ subtypes but not for instances of :class:`frozenset` or its subtypes. Return ``1`` if found and removed, ``0`` if not found (no action taken), and ``-1`` if an error is encountered. Does not raise :exc:`KeyError` for missing keys. Raise a - :exc:`TypeError` if the *key* is unhashable. Unlike the Python :meth:`~frozenset.discard` + :exc:`TypeError` if the *key* is unhashable. Unlike the Python :meth:`~set.discard` method, this function does not automatically convert unhashable sets into temporary frozensets. Raise :exc:`SystemError` if *set* is not an instance of :class:`set` or its subtype. diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 086da1a705c30f..3899e5b59d8852 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -4826,7 +4826,7 @@ other sequence-like behavior. There are currently two built-in set types, :class:`set` and :class:`frozenset`. The :class:`set` type is mutable --- the contents can be changed using methods -like :meth:`add ` and :meth:`remove `. +like :meth:`~set.add` and :meth:`~set.remove`. Since it is mutable, it has no hash value and cannot be used as either a dictionary key or as an element of another set. The :class:`frozenset` type is immutable and :term:`hashable` --- @@ -4848,164 +4848,172 @@ The constructors for both classes work the same: objects. If *iterable* is not specified, a new empty set is returned. - Sets can be created by several means: +Sets can be created by several means: - * Use a comma-separated list of elements within braces: ``{'jack', 'sjoerd'}`` - * Use a set comprehension: ``{c for c in 'abracadabra' if c not in 'abc'}`` - * Use the type constructor: ``set()``, ``set('foobar')``, ``set(['a', 'b', 'foo'])`` +* Use a comma-separated list of elements within braces: ``{'jack', 'sjoerd'}`` +* Use a set comprehension: ``{c for c in 'abracadabra' if c not in 'abc'}`` +* Use the type constructor: ``set()``, ``set('foobar')``, ``set(['a', 'b', 'foo'])`` - Instances of :class:`set` and :class:`frozenset` provide the following - operations: +Instances of :class:`set` and :class:`frozenset` provide the following +operations: - .. describe:: len(s) +.. describe:: len(s) - Return the number of elements in set *s* (cardinality of *s*). + Return the number of elements in set *s* (cardinality of *s*). - .. describe:: x in s +.. describe:: x in s - Test *x* for membership in *s*. + Test *x* for membership in *s*. - .. describe:: x not in s +.. describe:: x not in s - Test *x* for non-membership in *s*. + Test *x* for non-membership in *s*. - .. method:: isdisjoint(other, /) +.. method:: frozenset.isdisjoint(other, /) + set.isdisjoint(other, /) - Return ``True`` if the set has no elements in common with *other*. Sets are - disjoint if and only if their intersection is the empty set. + Return ``True`` if the set has no elements in common with *other*. Sets are + disjoint if and only if their intersection is the empty set. - .. method:: issubset(other, /) - set <= other +.. method:: frozenset.issubset(other, /) + set.issubset(other, /) +.. describe:: set <= other - Test whether every element in the set is in *other*. + Test whether every element in the set is in *other*. - .. method:: set < other +.. describe:: set < other - Test whether the set is a proper subset of *other*, that is, - ``set <= other and set != other``. + Test whether the set is a proper subset of *other*, that is, + ``set <= other and set != other``. - .. method:: issuperset(other, /) - set >= other +.. method:: frozenset.issuperset(other, /) + set.issuperset(other, /) +.. describe:: set >= other - Test whether every element in *other* is in the set. + Test whether every element in *other* is in the set. - .. method:: set > other +.. describe:: set > other - Test whether the set is a proper superset of *other*, that is, ``set >= - other and set != other``. + Test whether the set is a proper superset of *other*, that is, ``set >= + other and set != other``. - .. method:: union(*others) - set | other | ... +.. method:: frozenset.union(*others) + set.union(*others) +.. describe:: set | other | ... - Return a new set with elements from the set and all others. + Return a new set with elements from the set and all others. - .. method:: intersection(*others) - set & other & ... +.. method:: frozenset.intersection(*others) + set.intersection(*others) +.. describe:: set & other & ... - Return a new set with elements common to the set and all others. + Return a new set with elements common to the set and all others. - .. method:: difference(*others) - set - other - ... +.. method:: frozenset.difference(*others) + set.difference(*others) +.. describe:: set - other - ... - Return a new set with elements in the set that are not in the others. + Return a new set with elements in the set that are not in the others. - .. method:: symmetric_difference(other, /) - set ^ other +.. method:: frozenset.symmetric_difference(other, /) + set.symmetric_difference(other, /) +.. describe:: set ^ other - Return a new set with elements in either the set or *other* but not both. + Return a new set with elements in either the set or *other* but not both. - .. method:: copy() +.. method:: frozenset.copy() + set.copy() - Return a shallow copy of the set. + Return a shallow copy of the set. - Note, the non-operator versions of :meth:`union`, :meth:`intersection`, - :meth:`difference`, :meth:`symmetric_difference`, :meth:`issubset`, and - :meth:`issuperset` methods will accept any iterable as an argument. In - contrast, their operator based counterparts require their arguments to be - sets. This precludes error-prone constructions like ``set('abc') & 'cbs'`` - in favor of the more readable ``set('abc').intersection('cbs')``. +Note, the non-operator versions of :meth:`~frozenset.union`, +:meth:`~frozenset.intersection`, :meth:`~frozenset.difference`, :meth:`~frozenset.symmetric_difference`, :meth:`~frozenset.issubset`, and +:meth:`~frozenset.issuperset` methods will accept any iterable as an argument. In +contrast, their operator based counterparts require their arguments to be +sets. This precludes error-prone constructions like ``set('abc') & 'cbs'`` +in favor of the more readable ``set('abc').intersection('cbs')``. - Both :class:`set` and :class:`frozenset` support set to set comparisons. Two - sets are equal if and only if every element of each set is contained in the - other (each is a subset of the other). A set is less than another set if and - only if the first set is a proper subset of the second set (is a subset, but - is not equal). A set is greater than another set if and only if the first set - is a proper superset of the second set (is a superset, but is not equal). +Both :class:`set` and :class:`frozenset` support set to set comparisons. Two +sets are equal if and only if every element of each set is contained in the +other (each is a subset of the other). A set is less than another set if and +only if the first set is a proper subset of the second set (is a subset, but +is not equal). A set is greater than another set if and only if the first set +is a proper superset of the second set (is a superset, but is not equal). - Instances of :class:`set` are compared to instances of :class:`frozenset` - based on their members. For example, ``set('abc') == frozenset('abc')`` - returns ``True`` and so does ``set('abc') in set([frozenset('abc')])``. +Instances of :class:`set` are compared to instances of :class:`frozenset` +based on their members. For example, ``set('abc') == frozenset('abc')`` +returns ``True`` and so does ``set('abc') in set([frozenset('abc')])``. - The subset and equality comparisons do not generalize to a total ordering - function. For example, any two nonempty disjoint sets are not equal and are not - subsets of each other, so *all* of the following return ``False``: ``ab``. +The subset and equality comparisons do not generalize to a total ordering +function. For example, any two nonempty disjoint sets are not equal and are not +subsets of each other, so *all* of the following return ``False``: ``ab``. - Since sets only define partial ordering (subset relationships), the output of - the :meth:`list.sort` method is undefined for lists of sets. +Since sets only define partial ordering (subset relationships), the output of +the :meth:`list.sort` method is undefined for lists of sets. - Set elements, like dictionary keys, must be :term:`hashable`. +Set elements, like dictionary keys, must be :term:`hashable`. - Binary operations that mix :class:`set` instances with :class:`frozenset` - return the type of the first operand. For example: ``frozenset('ab') | - set('bc')`` returns an instance of :class:`frozenset`. +Binary operations that mix :class:`set` instances with :class:`frozenset` +return the type of the first operand. For example: ``frozenset('ab') | +set('bc')`` returns an instance of :class:`frozenset`. - The following table lists operations available for :class:`set` that do not - apply to immutable instances of :class:`frozenset`: +The following table lists operations available for :class:`set` that do not +apply to immutable instances of :class:`frozenset`: - .. method:: update(*others) - set |= other | ... +.. method:: set.update(*others) +.. describe:: set |= other | ... - Update the set, adding elements from all others. + Update the set, adding elements from all others. - .. method:: intersection_update(*others) - set &= other & ... +.. method:: set.intersection_update(*others) +.. describe:: set &= other & ... - Update the set, keeping only elements found in it and all others. + Update the set, keeping only elements found in it and all others. - .. method:: difference_update(*others) - set -= other | ... +.. method:: set.difference_update(*others) +.. describe:: set -= other | ... - Update the set, removing elements found in others. + Update the set, removing elements found in others. - .. method:: symmetric_difference_update(other, /) - set ^= other +.. method:: set.symmetric_difference_update(other, /) +.. describe:: set ^= other - Update the set, keeping only elements found in either set, but not in both. + Update the set, keeping only elements found in either set, but not in both. - .. method:: add(elem, /) +.. method:: set.add(elem, /) - Add element *elem* to the set. + Add element *elem* to the set. - .. method:: remove(elem, /) +.. method:: set.remove(elem, /) - Remove element *elem* from the set. Raises :exc:`KeyError` if *elem* is - not contained in the set. + Remove element *elem* from the set. Raises :exc:`KeyError` if *elem* is + not contained in the set. - .. method:: discard(elem, /) +.. method:: set.discard(elem, /) - Remove element *elem* from the set if it is present. + Remove element *elem* from the set if it is present. - .. method:: pop() +.. method:: set.pop() - Remove and return an arbitrary element from the set. Raises - :exc:`KeyError` if the set is empty. + Remove and return an arbitrary element from the set. Raises + :exc:`KeyError` if the set is empty. - .. method:: clear() +.. method:: set.clear() - Remove all elements from the set. + Remove all elements from the set. - Note, the non-operator versions of the :meth:`update`, - :meth:`intersection_update`, :meth:`difference_update`, and - :meth:`symmetric_difference_update` methods will accept any iterable as an - argument. +Note, the non-operator versions of the :meth:`~set.update`, +:meth:`~set.intersection_update`, :meth:`~set.difference_update`, and +:meth:`~set.symmetric_difference_update` methods will accept any iterable as an +argument. - Note, the *elem* argument to the :meth:`~object.__contains__`, - :meth:`remove`, and - :meth:`discard` methods may be a set. To support searching for an equivalent - frozenset, a temporary one is created from *elem*. +Note, the *elem* argument to the :meth:`~object.__contains__`, +:meth:`~set.remove`, and +:meth:`~set.discard` methods may be a set. To support searching for an equivalent +frozenset, a temporary one is created from *elem*. .. _typesmapping: diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index ebadbc215a0eed..5f79c6fe8f50ff 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -449,7 +449,7 @@ Sets These represent a mutable set. They are created by the built-in :func:`set` constructor and can be modified afterwards by several methods, such as - :meth:`add `. + :meth:`~set.add`. Frozen sets diff --git a/Doc/whatsnew/2.3.rst b/Doc/whatsnew/2.3.rst index b7e4e73f4ce4aa..f43692b3dce9e8 100644 --- a/Doc/whatsnew/2.3.rst +++ b/Doc/whatsnew/2.3.rst @@ -66,7 +66,7 @@ Here's a simple example:: The union and intersection of sets can be computed with the :meth:`~frozenset.union` and :meth:`~frozenset.intersection` methods; an alternative notation uses the bitwise operators ``&`` and ``|``. Mutable sets also have in-place versions of these methods, -:meth:`!union_update` and :meth:`~frozenset.intersection_update`. :: +:meth:`!union_update` and :meth:`~set.intersection_update`. :: >>> S1 = sets.Set([1,2,3]) >>> S2 = sets.Set([4,5,6]) @@ -87,7 +87,7 @@ It's also possible to take the symmetric difference of two sets. This is the set of all elements in the union that aren't in the intersection. Another way of putting it is that the symmetric difference contains all elements that are in exactly one set. Again, there's an alternative notation (``^``), and an -in-place version with the ungainly name :meth:`~frozenset.symmetric_difference_update`. :: +in-place version with the ungainly name :meth:`~set.symmetric_difference_update`. :: >>> S1 = sets.Set([1,2,3,4]) >>> S2 = sets.Set([3,4,5,6]) From 706fdda8b360120a25b272898df40c8913381723 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 5 Dec 2025 16:24:35 +0200 Subject: [PATCH 264/638] gh-141370: Fix undefined behavior when using Py_ABS() (GH-141548) Co-authored-by: Sergey B Kirpichev --- Include/pymacro.h | 6 ++++++ Lib/test/test_bytes.py | 11 +++++++++++ Lib/test/test_marshal.py | 5 +++++ Lib/test/test_memoryview.py | 19 +++++++++++++++++++ Python/marshal.c | 2 +- Python/pystrhex.c | 3 +-- 6 files changed, 43 insertions(+), 3 deletions(-) diff --git a/Include/pymacro.h b/Include/pymacro.h index 857cdf12db9bf2..7ecce44a0d2a42 100644 --- a/Include/pymacro.h +++ b/Include/pymacro.h @@ -116,6 +116,12 @@ /* Absolute value of the number x */ #define Py_ABS(x) ((x) < 0 ? -(x) : (x)) +/* Safer implementation that avoids an undefined behavior for the minimal + value of the signed integer type if its absolute value is larger than + the maximal value of the signed integer type (in the two's complement + representations, which is common). + */ +#define _Py_ABS_CAST(T, x) ((x) >= 0 ? ((T) (x)) : ((T) (((T) -((x) + 1)) + 1u))) #define _Py_XSTRINGIFY(x) #x diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index a6cf899fa51e75..a55ec6cf3b8353 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -549,6 +549,17 @@ def test_hex_separator_basics(self): self.assertEqual(three_bytes.hex(':', 2), 'b9:01ef') self.assertEqual(three_bytes.hex(':', 1), 'b9:01:ef') self.assertEqual(three_bytes.hex('*', -2), 'b901*ef') + self.assertEqual(three_bytes.hex(sep=':', bytes_per_sep=2), 'b9:01ef') + self.assertEqual(three_bytes.hex(sep='*', bytes_per_sep=-2), 'b901*ef') + for bytes_per_sep in 3, -3, 2**31-1, -(2**31-1): + with self.subTest(bytes_per_sep=bytes_per_sep): + self.assertEqual(three_bytes.hex(':', bytes_per_sep), 'b901ef') + for bytes_per_sep in 2**31, -2**31, 2**1000, -2**1000: + with self.subTest(bytes_per_sep=bytes_per_sep): + try: + self.assertEqual(three_bytes.hex(':', bytes_per_sep), 'b901ef') + except OverflowError: + pass value = b'{s\005\000\000\000worldi\002\000\000\000s\005\000\000\000helloi\001\000\000\0000' self.assertEqual(value.hex('.', 8), '7b7305000000776f.726c646902000000.730500000068656c.6c6f690100000030') diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index 8b1fb0eba1f8b6..662bdfccc79125 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -43,6 +43,11 @@ def test_ints(self): for expected in (-n, n): self.helper(expected) n = n >> 1 + n = 1 << 100 + while n: + for expected in (-n, -n+1, n-1, n): + self.helper(expected) + n = n >> 1 def test_int64(self): # Simulate int marshaling with TYPE_INT64. diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py index 64f440f180bbf0..1bd58eb6408833 100644 --- a/Lib/test/test_memoryview.py +++ b/Lib/test/test_memoryview.py @@ -600,6 +600,25 @@ def test_memoryview_hex(self): m2 = m1[::-1] self.assertEqual(m2.hex(), '30' * 200000) + def test_memoryview_hex_separator(self): + x = bytes(range(97, 102)) + m1 = memoryview(x) + m2 = m1[::-1] + self.assertEqual(m2.hex(':'), '65:64:63:62:61') + self.assertEqual(m2.hex(':', 2), '65:6463:6261') + self.assertEqual(m2.hex(':', -2), '6564:6362:61') + self.assertEqual(m2.hex(sep=':', bytes_per_sep=2), '65:6463:6261') + self.assertEqual(m2.hex(sep=':', bytes_per_sep=-2), '6564:6362:61') + for bytes_per_sep in 5, -5, 2**31-1, -(2**31-1): + with self.subTest(bytes_per_sep=bytes_per_sep): + self.assertEqual(m2.hex(':', bytes_per_sep), '6564636261') + for bytes_per_sep in 2**31, -2**31, 2**1000, -2**1000: + with self.subTest(bytes_per_sep=bytes_per_sep): + try: + self.assertEqual(m2.hex(':', bytes_per_sep), '6564636261') + except OverflowError: + pass + def test_copy(self): m = memoryview(b'abc') with self.assertRaises(TypeError): diff --git a/Python/marshal.c b/Python/marshal.c index 8b56de6575559c..69d6dd7cf0f802 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -310,7 +310,7 @@ w_PyLong(const PyLongObject *ob, char flag, WFILE *p) } if (!long_export.digits) { int8_t sign = long_export.value < 0 ? -1 : 1; - uint64_t abs_value = Py_ABS(long_export.value); + uint64_t abs_value = _Py_ABS_CAST(uint64_t, long_export.value); uint64_t d = abs_value; long l = 0; diff --git a/Python/pystrhex.c b/Python/pystrhex.c index 38484f5a7d4227..af2f5c5dce5fca 100644 --- a/Python/pystrhex.c +++ b/Python/pystrhex.c @@ -42,8 +42,7 @@ static PyObject *_Py_strhex_impl(const char* argbuf, const Py_ssize_t arglen, else { bytes_per_sep_group = 0; } - - unsigned int abs_bytes_per_sep = Py_ABS(bytes_per_sep_group); + unsigned int abs_bytes_per_sep = _Py_ABS_CAST(unsigned int, bytes_per_sep_group); Py_ssize_t resultlen = 0; if (bytes_per_sep_group && arglen > 0) { /* How many sep characters we'll be inserting. */ From 100c726d9895ef26d0d279ae585c0228c0d8529f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 5 Dec 2025 18:09:20 +0200 Subject: [PATCH 265/638] Add explanation comments for tests for overlapped ZIP entries (GH-137152) --- Lib/test/test_zipfile/test_core.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index 1edb5dde998658..6887a5e5cc4d18 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -2531,6 +2531,10 @@ def test_decompress_without_3rd_party_library(self): @requires_zlib() def test_full_overlap_different_names(self): + # The ZIP file contains two central directory entries with + # different names which refer to the same local header. + # The name of the local header matches the name of the first + # central directory entry. data = ( b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e' b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00b\xed' @@ -2560,6 +2564,10 @@ def test_full_overlap_different_names(self): @requires_zlib() def test_full_overlap_different_names2(self): + # The ZIP file contains two central directory entries with + # different names which refer to the same local header. + # The name of the local header matches the name of the second + # central directory entry. data = ( b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e' b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00a\xed' @@ -2591,6 +2599,8 @@ def test_full_overlap_different_names2(self): @requires_zlib() def test_full_overlap_same_name(self): + # The ZIP file contains two central directory entries with + # the same name which refer to the same local header. data = ( b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e' b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00a\xed' @@ -2623,6 +2633,8 @@ def test_full_overlap_same_name(self): @requires_zlib() def test_quoted_overlap(self): + # The ZIP file contains two files. The second local header + # is contained in the range of the first file. data = ( b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05Y\xfc' b'8\x044\x00\x00\x00(\x04\x00\x00\x01\x00\x00\x00a\x00' @@ -2654,6 +2666,7 @@ def test_quoted_overlap(self): @requires_zlib() def test_overlap_with_central_dir(self): + # The local header offset is equal to the central directory offset. data = ( b'PK\x01\x02\x14\x03\x14\x00\x00\x00\x08\x00G_|Z' b'\xe2\x1e8\xbb\x0b\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00' @@ -2668,11 +2681,15 @@ def test_overlap_with_central_dir(self): self.assertEqual(zi.header_offset, 0) self.assertEqual(zi.compress_size, 11) self.assertEqual(zi.file_size, 1033) + # Found central directory signature PK\x01\x02 instead of + # local header signature PK\x03\x04. with self.assertRaisesRegex(zipfile.BadZipFile, 'Bad magic number'): zipf.read('a') @requires_zlib() def test_overlap_with_archive_comment(self): + # The local header is written after the central directory, + # in the archive comment. data = ( b'PK\x01\x02\x14\x03\x14\x00\x00\x00\x08\x00G_|Z' b'\xe2\x1e8\xbb\x0b\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00' From 4b145297301fbcb18461a4e933a4188b2515fad4 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Fri, 5 Dec 2025 08:21:31 -0800 Subject: [PATCH 266/638] GH-139862: Remove `color` from HelpFormatter (#142274) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Lib/argparse.py | 2 -- .../next/Library/2025-12-04-23-24-24.gh-issue-139862.NBfsD4.rst | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-12-04-23-24-24.gh-issue-139862.NBfsD4.rst diff --git a/Lib/argparse.py b/Lib/argparse.py index 55ecdadd8c9398..41467707d393c0 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -166,7 +166,6 @@ def __init__( indent_increment=2, max_help_position=24, width=None, - color=True, ): # default setting for width if width is None: @@ -174,7 +173,6 @@ def __init__( width = shutil.get_terminal_size().columns width -= 2 - self._set_color(color) self._prog = prog self._indent_increment = indent_increment self._max_help_position = min(max_help_position, diff --git a/Misc/NEWS.d/next/Library/2025-12-04-23-24-24.gh-issue-139862.NBfsD4.rst b/Misc/NEWS.d/next/Library/2025-12-04-23-24-24.gh-issue-139862.NBfsD4.rst new file mode 100644 index 00000000000000..2bee8881a75749 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-04-23-24-24.gh-issue-139862.NBfsD4.rst @@ -0,0 +1 @@ +Remove ``color`` parameter from :class:`!argparse.HelpFormatter` constructor. Color is controlled by :class:`~argparse.ArgumentParser`. From 4085ff7b32f91bad7d821e5564d8565c5928f7d1 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Fri, 5 Dec 2025 08:47:50 -0800 Subject: [PATCH 267/638] GH-142267: Cache formatter to avoid repeated `_set_color` calls (#142268) --- Lib/argparse.py | 18 ++++++++++++++---- ...5-12-04-23-26-12.gh-issue-142267.yOM6fP.rst | 1 + 2 files changed, 15 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-12-04-23-26-12.gh-issue-142267.yOM6fP.rst diff --git a/Lib/argparse.py b/Lib/argparse.py index 41467707d393c0..10393b6a02b0be 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -1568,8 +1568,8 @@ def add_argument(self, *args, **kwargs): f'instance of it must be passed') # raise an error if the metavar does not match the type - if hasattr(self, "_get_formatter"): - formatter = self._get_formatter() + if hasattr(self, "_get_validation_formatter"): + formatter = self._get_validation_formatter() try: formatter._format_args(action, None) except TypeError: @@ -1763,8 +1763,8 @@ def _handle_conflict_resolve(self, action, conflicting_actions): action.container._remove_action(action) def _check_help(self, action): - if action.help and hasattr(self, "_get_formatter"): - formatter = self._get_formatter() + if action.help and hasattr(self, "_get_validation_formatter"): + formatter = self._get_validation_formatter() try: formatter._expand_help(action) except (ValueError, TypeError, KeyError) as exc: @@ -1919,6 +1919,9 @@ def __init__(self, self.suggest_on_error = suggest_on_error self.color = color + # Cached formatter for validation (avoids repeated _set_color calls) + self._cached_formatter = None + add_group = self.add_argument_group self._positionals = add_group(_('positional arguments')) self._optionals = add_group(_('options')) @@ -2750,6 +2753,13 @@ def _get_formatter(self): formatter._set_color(self.color) return formatter + def _get_validation_formatter(self): + # Return cached formatter for read-only validation operations + # (_expand_help and _format_args). Avoids repeated slow _set_color calls. + if self._cached_formatter is None: + self._cached_formatter = self._get_formatter() + return self._cached_formatter + # ===================== # Help-printing methods # ===================== diff --git a/Misc/NEWS.d/next/Library/2025-12-04-23-26-12.gh-issue-142267.yOM6fP.rst b/Misc/NEWS.d/next/Library/2025-12-04-23-26-12.gh-issue-142267.yOM6fP.rst new file mode 100644 index 00000000000000..f46e82105fc2f5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-04-23-26-12.gh-issue-142267.yOM6fP.rst @@ -0,0 +1 @@ +Improve :mod:`argparse` performance by caching the formatter used for argument validation. From 59f247e43bc93c607a5793c220bcaafb712cf542 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 5 Dec 2025 19:17:01 +0200 Subject: [PATCH 268/638] gh-115952: Fix a potential virtual memory allocation denial of service in pickle (GH-119204) Loading a small data which does not even involve arbitrary code execution could consume arbitrary large amount of memory. There were three issues: * PUT and LONG_BINPUT with large argument (the C implementation only). Since the memo is implemented in C as a continuous dynamic array, a single opcode can cause its resizing to arbitrary size. Now the sparsity of memo indices is limited. * BINBYTES, BINBYTES8 and BYTEARRAY8 with large argument. They allocated the bytes or bytearray object of the specified size before reading into it. Now they read very large data by chunks. * BINSTRING, BINUNICODE, LONG4, BINUNICODE8 and FRAME with large argument. They read the whole data by calling the read() method of the underlying file object, which usually allocates the bytes object of the specified size before reading into it. Now they read very large data by chunks. Also add comprehensive benchmark suite to measure performance and memory impact of chunked reading optimization in PR #119204. Features: - Normal mode: benchmarks legitimate pickles (time/memory metrics) - Antagonistic mode: tests malicious pickles (DoS protection) - Baseline comparison: side-by-side comparison of two Python builds - Support for truncated data and sparse memo attack vectors Co-Authored-By: Claude Sonnet 4.5 Co-authored-by: Gregory P. Smith --- Lib/pickle.py | 34 +- Lib/test/pickletester.py | 167 ++- Lib/test/test_pickle.py | 8 +- ...-05-20-12-35-52.gh-issue-115952.J6n_Kf.rst | 7 + Modules/_pickle.c | 277 +++-- Tools/picklebench/README.md | 232 ++++ Tools/picklebench/memory_dos_impact.py | 1069 +++++++++++++++++ 7 files changed, 1692 insertions(+), 102 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-20-12-35-52.gh-issue-115952.J6n_Kf.rst create mode 100644 Tools/picklebench/README.md create mode 100755 Tools/picklebench/memory_dos_impact.py diff --git a/Lib/pickle.py b/Lib/pickle.py index 729c215514ad24..f3025776623d2c 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -189,6 +189,11 @@ def __init__(self, value): __all__.extend(x for x in dir() if x.isupper() and not x.startswith('_')) +# Data larger than this will be read in chunks, to prevent extreme +# overallocation. +_MIN_READ_BUF_SIZE = (1 << 20) + + class _Framer: _FRAME_SIZE_MIN = 4 @@ -287,7 +292,7 @@ def read(self, n): "pickle exhausted before end of frame") return data else: - return self.file_read(n) + return self._chunked_file_read(n) def readline(self): if self.current_frame: @@ -302,11 +307,23 @@ def readline(self): else: return self.file_readline() + def _chunked_file_read(self, size): + cursize = min(size, _MIN_READ_BUF_SIZE) + b = self.file_read(cursize) + while cursize < size and len(b) == cursize: + delta = min(cursize, size - cursize) + b += self.file_read(delta) + cursize += delta + return b + def load_frame(self, frame_size): if self.current_frame and self.current_frame.read() != b'': raise UnpicklingError( "beginning of a new frame before end of current frame") - self.current_frame = io.BytesIO(self.file_read(frame_size)) + data = self._chunked_file_read(frame_size) + if len(data) < frame_size: + raise EOFError + self.current_frame = io.BytesIO(data) # Tools used for pickling. @@ -1496,12 +1513,17 @@ def load_binbytes8(self): dispatch[BINBYTES8[0]] = load_binbytes8 def load_bytearray8(self): - len, = unpack(' maxsize: + size, = unpack(' maxsize: raise UnpicklingError("BYTEARRAY8 exceeds system's maximum size " "of %d bytes" % maxsize) - b = bytearray(len) - self.readinto(b) + cursize = min(size, _MIN_READ_BUF_SIZE) + b = bytearray(cursize) + if self.readinto(b) == cursize: + while cursize < size and len(b) == cursize: + delta = min(cursize, size - cursize) + b += self.read(delta) + cursize += delta self.append(b) dispatch[BYTEARRAY8[0]] = load_bytearray8 diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index e3663e44546ded..4e3468bfcde9c3 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -74,6 +74,15 @@ def count_opcode(code, pickle): def identity(x): return x +def itersize(start, stop): + # Produce geometrical increasing sequence from start to stop + # (inclusively) for tests. + size = start + while size < stop: + yield size + size <<= 1 + yield stop + class UnseekableIO(io.BytesIO): def peek(self, *args): @@ -853,9 +862,8 @@ def assert_is_copy(self, obj, objcopy, msg=None): self.assertEqual(getattr(obj, slot, None), getattr(objcopy, slot, None), msg=msg) - def check_unpickling_error(self, errors, data): - with self.subTest(data=data), \ - self.assertRaises(errors): + def check_unpickling_error_strict(self, errors, data): + with self.assertRaises(errors): try: self.loads(data) except BaseException as exc: @@ -864,6 +872,10 @@ def check_unpickling_error(self, errors, data): (data, exc.__class__.__name__, exc)) raise + def check_unpickling_error(self, errors, data): + with self.subTest(data=data): + self.check_unpickling_error_strict(errors, data) + def test_load_from_data0(self): self.assert_is_copy(self._testdata, self.loads(DATA0)) @@ -1150,6 +1162,155 @@ def test_negative_32b_binput(self): dumped = b'\x80\x03X\x01\x00\x00\x00ar\xff\xff\xff\xff.' self.check_unpickling_error(ValueError, dumped) + def test_too_large_put(self): + # Test that PUT with large id does not cause allocation of + # too large memo table. The C implementation uses a dict-based memo + # for sparse indices (when idx > memo_len * 2) instead of allocating + # a massive array. This test verifies large sparse indices work without + # causing memory exhaustion. + # + # The following simple pickle creates an empty list, memoizes it + # using a large index, then loads it back on the stack, builds + # a tuple containing 2 identical empty lists and returns it. + data = lambda n: (b'((lp' + str(n).encode() + b'\n' + + b'g' + str(n).encode() + b'\nt.') + # 0: ( MARK + # 1: ( MARK + # 2: l LIST (MARK at 1) + # 3: p PUT 1000000000000 + # 18: g GET 1000000000000 + # 33: t TUPLE (MARK at 0) + # 34: . STOP + for idx in [10**6, 10**9, 10**12]: + if idx > sys.maxsize: + continue + self.assertEqual(self.loads(data(idx)), ([],)*2) + + def test_too_large_long_binput(self): + # Test that LONG_BINPUT with large id does not cause allocation of + # too large memo table. The C implementation uses a dict-based memo + # for sparse indices (when idx > memo_len * 2) instead of allocating + # a massive array. This test verifies large sparse indices work without + # causing memory exhaustion. + # + # The following simple pickle creates an empty list, memoizes it + # using a large index, then loads it back on the stack, builds + # a tuple containing 2 identical empty lists and returns it. + data = lambda n: (b'(]r' + struct.pack(' sys.maxsize')) + + def test_truncated_large_binunicode8(self): + data = lambda size: b'\x8d' + struct.pack('readinto) { + /* readinto() not supported on file-like object, fall back to read() + * and copy into destination buffer (bpo-39681) */ + PyObject* len = PyLong_FromSsize_t(n); + if (len == NULL) { + return -1; + } + PyObject* data = _Pickle_FastCall(self->read, len); + if (data == NULL) { + return -1; + } + if (!PyBytes_Check(data)) { + PyErr_Format(PyExc_ValueError, + "read() returned non-bytes object (%R)", + Py_TYPE(data)); + Py_DECREF(data); + return -1; + } + Py_ssize_t read_size = PyBytes_GET_SIZE(data); + if (read_size < n) { + Py_DECREF(data); + return bad_readline(state); + } + memcpy(buf, PyBytes_AS_STRING(data), n); + Py_DECREF(data); + return n; + } + + /* Call readinto() into user buffer */ + PyObject *buf_obj = PyMemoryView_FromMemory(buf, n, PyBUF_WRITE); + if (buf_obj == NULL) { + return -1; + } + PyObject *read_size_obj = _Pickle_FastCall(self->readinto, buf_obj); + if (read_size_obj == NULL) { + return -1; + } + Py_ssize_t read_size = PyLong_AsSsize_t(read_size_obj); + Py_DECREF(read_size_obj); + + if (read_size < 0) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ValueError, + "readinto() returned negative size"); + } + return -1; + } + if (read_size < n) { + return bad_readline(state); + } + return n; +} + /* If reading from a file, we need to only pull the bytes we need, since there may be multiple pickle objects arranged contiguously in the same input buffer. @@ -1262,7 +1326,7 @@ static const Py_ssize_t READ_WHOLE_LINE = -1; causing the Unpickler to go back to the file for more data. Use the returned size to tell you how much data you can process. */ static Py_ssize_t -_Unpickler_ReadFromFile(UnpicklerObject *self, Py_ssize_t n) +_Unpickler_ReadFromFile(PickleState *state, UnpicklerObject *self, Py_ssize_t n) { PyObject *data; Py_ssize_t read_size; @@ -1274,6 +1338,9 @@ _Unpickler_ReadFromFile(UnpicklerObject *self, Py_ssize_t n) if (n == READ_WHOLE_LINE) { data = PyObject_CallNoArgs(self->readline); + if (data == NULL) { + return -1; + } } else { PyObject *len; @@ -1302,13 +1369,29 @@ _Unpickler_ReadFromFile(UnpicklerObject *self, Py_ssize_t n) return n; } } - len = PyLong_FromSsize_t(n); + Py_ssize_t cursize = Py_MIN(n, MIN_READ_BUF_SIZE); + len = PyLong_FromSsize_t(cursize); if (len == NULL) return -1; data = _Pickle_FastCall(self->read, len); + if (data == NULL) { + return -1; + } + while (cursize < n) { + Py_ssize_t prevsize = cursize; + // geometrically double the chunk size to avoid CPU DoS + cursize += Py_MIN(cursize, n - cursize); + if (_PyBytes_Resize(&data, cursize) < 0) { + return -1; + } + if (_Unpickler_ReadIntoFromFile(state, self, + PyBytes_AS_STRING(data) + prevsize, cursize - prevsize) < 0) + { + Py_DECREF(data); + return -1; + } + } } - if (data == NULL) - return -1; read_size = _Unpickler_SetStringInput(self, data); Py_DECREF(data); @@ -1335,7 +1418,7 @@ _Unpickler_ReadImpl(UnpicklerObject *self, PickleState *st, char **s, Py_ssize_t return bad_readline(st); /* Extend the buffer to satisfy desired size */ - num_read = _Unpickler_ReadFromFile(self, n); + num_read = _Unpickler_ReadFromFile(st, self, n); if (num_read < 0) return -1; if (num_read < n) @@ -1382,57 +1465,7 @@ _Unpickler_ReadInto(PickleState *state, UnpicklerObject *self, char *buf, return -1; } - if (!self->readinto) { - /* readinto() not supported on file-like object, fall back to read() - * and copy into destination buffer (bpo-39681) */ - PyObject* len = PyLong_FromSsize_t(n); - if (len == NULL) { - return -1; - } - PyObject* data = _Pickle_FastCall(self->read, len); - if (data == NULL) { - return -1; - } - if (!PyBytes_Check(data)) { - PyErr_Format(PyExc_ValueError, - "read() returned non-bytes object (%R)", - Py_TYPE(data)); - Py_DECREF(data); - return -1; - } - Py_ssize_t read_size = PyBytes_GET_SIZE(data); - if (read_size < n) { - Py_DECREF(data); - return bad_readline(state); - } - memcpy(buf, PyBytes_AS_STRING(data), n); - Py_DECREF(data); - return n; - } - - /* Call readinto() into user buffer */ - PyObject *buf_obj = PyMemoryView_FromMemory(buf, n, PyBUF_WRITE); - if (buf_obj == NULL) { - return -1; - } - PyObject *read_size_obj = _Pickle_FastCall(self->readinto, buf_obj); - if (read_size_obj == NULL) { - return -1; - } - Py_ssize_t read_size = PyLong_AsSsize_t(read_size_obj); - Py_DECREF(read_size_obj); - - if (read_size < 0) { - if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_ValueError, - "readinto() returned negative size"); - } - return -1; - } - if (read_size < n) { - return bad_readline(state); - } - return n; + return _Unpickler_ReadIntoFromFile(state, self, buf, n); } /* Read `n` bytes from the unpickler's data source, storing the result in `*s`. @@ -1492,7 +1525,7 @@ _Unpickler_Readline(PickleState *state, UnpicklerObject *self, char **result) if (!self->read) return bad_readline(state); - num_read = _Unpickler_ReadFromFile(self, READ_WHOLE_LINE); + num_read = _Unpickler_ReadFromFile(state, self, READ_WHOLE_LINE); if (num_read < 0) return -1; if (num_read == 0 || self->input_buffer[num_read - 1] != '\n') @@ -1525,12 +1558,35 @@ _Unpickler_ResizeMemoList(UnpicklerObject *self, size_t new_size) /* Returns NULL if idx is out of bounds. */ static PyObject * -_Unpickler_MemoGet(UnpicklerObject *self, size_t idx) +_Unpickler_MemoGet(PickleState *st, UnpicklerObject *self, size_t idx) { - if (idx >= self->memo_size) - return NULL; - - return self->memo[idx]; + PyObject *value; + if (idx < self->memo_size) { + value = self->memo[idx]; + if (value != NULL) { + return value; + } + } + if (self->memo_dict != NULL) { + PyObject *key = PyLong_FromSize_t(idx); + if (key == NULL) { + return NULL; + } + if (idx < self->memo_size) { + (void)PyDict_Pop(self->memo_dict, key, &value); + // Migrate dict entry to array for faster future access + self->memo[idx] = value; + } + else { + value = PyDict_GetItemWithError(self->memo_dict, key); + } + Py_DECREF(key); + if (value != NULL || PyErr_Occurred()) { + return value; + } + } + PyErr_Format(st->UnpicklingError, "Memo value not found at index %zd", idx); + return NULL; } /* Returns -1 (with an exception set) on failure, 0 on success. @@ -1541,6 +1597,27 @@ _Unpickler_MemoPut(UnpicklerObject *self, size_t idx, PyObject *value) PyObject *old_item; if (idx >= self->memo_size) { + if (idx > self->memo_len * 2) { + /* The memo keys are too sparse. Use a dict instead of + * a continuous array for the memo. */ + if (self->memo_dict == NULL) { + self->memo_dict = PyDict_New(); + if (self->memo_dict == NULL) { + return -1; + } + } + PyObject *key = PyLong_FromSize_t(idx); + if (key == NULL) { + return -1; + } + + if (PyDict_SetItem(self->memo_dict, key, value) < 0) { + Py_DECREF(key); + return -1; + } + Py_DECREF(key); + return 0; + } if (_Unpickler_ResizeMemoList(self, idx * 2) < 0) return -1; assert(idx < self->memo_size); @@ -1610,6 +1687,7 @@ _Unpickler_New(PyObject *module) self->memo = memo; self->memo_size = MEMO_SIZE; self->memo_len = 0; + self->memo_dict = NULL; self->persistent_load = NULL; self->persistent_load_attr = NULL; memset(&self->buffer, 0, sizeof(Py_buffer)); @@ -5582,13 +5660,28 @@ load_counted_binbytes(PickleState *state, UnpicklerObject *self, int nbytes) return -1; } - bytes = PyBytes_FromStringAndSize(NULL, size); - if (bytes == NULL) - return -1; - if (_Unpickler_ReadInto(state, self, PyBytes_AS_STRING(bytes), size) < 0) { - Py_DECREF(bytes); + Py_ssize_t cursize = Py_MIN(size, MIN_READ_BUF_SIZE); + Py_ssize_t prevsize = 0; + bytes = PyBytes_FromStringAndSize(NULL, cursize); + if (bytes == NULL) { return -1; } + while (1) { + if (_Unpickler_ReadInto(state, self, + PyBytes_AS_STRING(bytes) + prevsize, cursize - prevsize) < 0) + { + Py_DECREF(bytes); + return -1; + } + if (cursize >= size) { + break; + } + prevsize = cursize; + cursize += Py_MIN(cursize, size - cursize); + if (_PyBytes_Resize(&bytes, cursize) < 0) { + return -1; + } + } PDATA_PUSH(self->stack, bytes, -1); return 0; @@ -5613,14 +5706,27 @@ load_counted_bytearray(PickleState *state, UnpicklerObject *self) return -1; } - bytearray = PyByteArray_FromStringAndSize(NULL, size); + Py_ssize_t cursize = Py_MIN(size, MIN_READ_BUF_SIZE); + Py_ssize_t prevsize = 0; + bytearray = PyByteArray_FromStringAndSize(NULL, cursize); if (bytearray == NULL) { return -1; } - char *str = PyByteArray_AS_STRING(bytearray); - if (_Unpickler_ReadInto(state, self, str, size) < 0) { - Py_DECREF(bytearray); - return -1; + while (1) { + if (_Unpickler_ReadInto(state, self, + PyByteArray_AS_STRING(bytearray) + prevsize, + cursize - prevsize) < 0) { + Py_DECREF(bytearray); + return -1; + } + if (cursize >= size) { + break; + } + prevsize = cursize; + cursize += Py_MIN(cursize, size - cursize); + if (PyByteArray_Resize(bytearray, cursize) < 0) { + return -1; + } } PDATA_PUSH(self->stack, bytearray, -1); @@ -6222,20 +6328,15 @@ load_get(PickleState *st, UnpicklerObject *self) if (key == NULL) return -1; idx = PyLong_AsSsize_t(key); + Py_DECREF(key); if (idx == -1 && PyErr_Occurred()) { - Py_DECREF(key); return -1; } - value = _Unpickler_MemoGet(self, idx); + value = _Unpickler_MemoGet(st, self, idx); if (value == NULL) { - if (!PyErr_Occurred()) { - PyErr_Format(st->UnpicklingError, "Memo value not found at index %ld", idx); - } - Py_DECREF(key); return -1; } - Py_DECREF(key); PDATA_APPEND(self->stack, value, -1); return 0; @@ -6253,13 +6354,8 @@ load_binget(PickleState *st, UnpicklerObject *self) idx = Py_CHARMASK(s[0]); - value = _Unpickler_MemoGet(self, idx); + value = _Unpickler_MemoGet(st, self, idx); if (value == NULL) { - PyObject *key = PyLong_FromSsize_t(idx); - if (key != NULL) { - PyErr_Format(st->UnpicklingError, "Memo value not found at index %ld", idx); - Py_DECREF(key); - } return -1; } @@ -6279,13 +6375,8 @@ load_long_binget(PickleState *st, UnpicklerObject *self) idx = calc_binsize(s, 4); - value = _Unpickler_MemoGet(self, idx); + value = _Unpickler_MemoGet(st, self, idx); if (value == NULL) { - PyObject *key = PyLong_FromSsize_t(idx); - if (key != NULL) { - PyErr_Format(st->UnpicklingError, "Memo value not found at index %ld", idx); - Py_DECREF(key); - } return -1; } @@ -7250,6 +7341,7 @@ Unpickler_clear(PyObject *op) self->buffer.buf = NULL; } + Py_CLEAR(self->memo_dict); _Unpickler_MemoCleanup(self); PyMem_Free(self->marks); self->marks = NULL; @@ -7286,6 +7378,7 @@ Unpickler_traverse(PyObject *op, visitproc visit, void *arg) Py_VISIT(self->persistent_load); Py_VISIT(self->persistent_load_attr); Py_VISIT(self->buffers); + Py_VISIT(self->memo_dict); PyObject **memo = self->memo; if (memo) { Py_ssize_t i = self->memo_size; diff --git a/Tools/picklebench/README.md b/Tools/picklebench/README.md new file mode 100644 index 00000000000000..7d52485c386350 --- /dev/null +++ b/Tools/picklebench/README.md @@ -0,0 +1,232 @@ +# Pickle Chunked Reading Benchmark + +This benchmark measures the performance impact of the chunked reading optimization in GH PR #119204 for the pickle module. + +## What This Tests + +The PR adds chunked reading (1MB chunks) to prevent memory exhaustion when unpickling large objects: +- **BINBYTES8** - Large bytes objects (protocol 4+) +- **BINUNICODE8** - Large strings (protocol 4+) +- **BYTEARRAY8** - Large bytearrays (protocol 5) +- **FRAME** - Large frames +- **LONG4** - Large integers +- An antagonistic mode that tests using memory denial of service inducing malicious pickles. + +## Quick Start + +```bash +# Run full benchmark suite (1MiB → 200MiB, takes several minutes) +build/python Tools/picklebench/memory_dos_impact.py + +# Test just a few sizes (quick test: 1, 10, 50 MiB) +build/python Tools/picklebench/memory_dos_impact.py --sizes 1 10 50 + +# Test smaller range for faster results +build/python Tools/picklebench/memory_dos_impact.py --sizes 1 5 10 + +# Output as markdown for reports +build/python Tools/picklebench/memory_dos_impact.py --format markdown > results.md + +# Test with protocol 4 instead of 5 +build/python Tools/picklebench/memory_dos_impact.py --protocol 4 +``` + +**Note:** Sizes are specified in MiB. Use `--sizes 1 2 5` for 1MiB, 2MiB, 5MiB objects. + +## Antagonistic Mode (DoS Protection Test) + +The `--antagonistic` flag tests **malicious pickles** that demonstrate the memory DoS protection: + +```bash +# Quick DoS protection test (claims 10, 50, 100 MB but provides 1KB) +build/python Tools/picklebench/memory_dos_impact.py --antagonistic --sizes 10 50 100 + +# Full DoS test (default: 10, 50, 100, 500, 1000, 5000 MB claimed) +build/python Tools/picklebench/memory_dos_impact.py --antagonistic +``` + +### What This Tests + +Unlike normal benchmarks that test **legitimate pickles**, antagonistic mode tests: +- **Truncated BINBYTES8**: Claims 100MB but provides only 1KB (will fail to unpickle) +- **Truncated BINUNICODE8**: Same for strings +- **Truncated BYTEARRAY8**: Same for bytearrays +- **Sparse memo attacks**: PUT at index 1 billion (would allocate huge array before PR) + +**Key difference:** +- **Normal mode**: Tests real data, shows ~5% time overhead +- **Antagonistic mode**: Tests malicious data, shows ~99% memory savings + +### Expected Results + +``` +100MB Claimed (actual: 1KB) + binbytes8_100MB_claim + Peak memory: 1.00 MB (claimed: 100 MB, saved: 99.00 MB, 99.0%) + Error: UnpicklingError ← Expected! + +Summary: + Average claimed: 126.2 MB + Average peak: 0.54 MB + Average saved: 125.7 MB (99.6% reduction) +Protection Status: ✓ Memory DoS attacks mitigated by chunked reading +``` + +**Before PR**: Would allocate full claimed size (100MB+), potentially crash +**After PR**: Allocates 1MB chunks, fails fast with minimal memory + +This demonstrates the **security improvement** - protection against memory exhaustion attacks. + +## Before/After Comparison + +The benchmark includes an automatic comparison feature that runs the same tests on both a baseline and current Python build. + +### Option 1: Automatic Comparison (Recommended) + +Build both versions, then use `--baseline` to automatically compare: + +```bash +# Build the baseline (main branch without PR) +git checkout main +mkdir -p build-main +cd build-main && ../configure && make -j $(nproc) && cd .. + +# Build the current version (with PR) +git checkout unpickle-overallocate +mkdir -p build +cd build && ../configure && make -j $(nproc) && cd .. + +# Run automatic comparison (quick test with a few sizes) +build/python Tools/picklebench/memory_dos_impact.py \ + --baseline build-main/python \ + --sizes 1 10 50 + +# Full comparison (all default sizes) +build/python Tools/picklebench/memory_dos_impact.py \ + --baseline build-main/python +``` + +The comparison output shows: +- Side-by-side metrics (Current vs Baseline) +- Percentage change for time and memory +- Overall summary statistics + +### Interpreting Comparison Results + +- **Time change**: Small positive % is expected (chunking adds overhead, typically 5-10%) +- **Memory change**: Negative % is good (chunking saves memory, especially for large objects) +- **Trade-off**: Slightly slower but much safer against memory exhaustion attacks + +### Option 2: Manual Comparison + +Save results separately and compare manually: + +```bash +# Baseline results +build-main/python Tools/picklebench/memory_dos_impact.py --format json > baseline.json + +# Current results +build/python Tools/picklebench/memory_dos_impact.py --format json > current.json + +# Manual comparison +diff -y <(jq '.' baseline.json) <(jq '.' current.json) +``` + +## Understanding the Results + +### Critical Sizes + +The default test suite includes: +- **< 1MiB (999,000 bytes)**: No chunking, allocates full size upfront +- **= 1MiB (1,048,576 bytes)**: Threshold, chunking just starts +- **> 1MiB (1,048,577 bytes)**: Chunked reading engaged +- **1, 2, 5, 10MiB**: Show scaling behavior with chunking +- **20, 50, 100, 200MiB**: Stress test large object handling + +**Note:** The full suite may require more than 16GiB of RAM. + +### Key Metrics + +- **Time (mean)**: Average unpickling time - should be similar before/after +- **Time (stdev)**: Consistency - lower is better +- **Peak Memory**: Maximum memory during unpickling - **expected to be LOWER after PR** +- **Pickle Size**: Size of the serialized data on disk + +### Test Types + +| Test | What It Stresses | +|------|------------------| +| `bytes_*` | BINBYTES8 opcode, raw binary data | +| `string_ascii_*` | BINUNICODE8 with simple ASCII | +| `string_utf8_*` | BINUNICODE8 with multibyte UTF-8 (€ chars) | +| `bytearray_*` | BYTEARRAY8 opcode (protocol 5) | +| `list_large_items_*` | Multiple chunked reads in sequence | +| `dict_large_values_*` | Chunking in dict deserialization | +| `nested_*` | Realistic mixed data structures | +| `tuple_*` | Immutable structures | + +## Expected Results + +### Before PR (main branch) +- Single large allocation per object +- Risk of memory exhaustion with malicious pickles + +### After PR (unpickle-overallocate branch) +- Chunked allocation (1MB at a time) +- **Slightly higher CPU time** (multiple allocations + resizing) +- **Significantly lower peak memory** (no large pre-allocation) +- Protection against DoS via memory exhaustion + +## Advanced Usage + +### Test Specific Sizes + +```bash +# Test only 5MiB and 10MiB objects +build/python Tools/picklebench/memory_dos_impact.py --sizes 5 10 + +# Test large objects: 50, 100, 200 MiB +build/python Tools/picklebench/memory_dos_impact.py --sizes 50 100 200 +``` + +### More Iterations for Stable Timing + +```bash +# Run 10 iterations per test for better statistics +build/python Tools/picklebench/memory_dos_impact.py --iterations 10 --sizes 1 10 +``` + +### JSON Output for Analysis + +```bash +# Generate JSON for programmatic analysis +build/python Tools/picklebench/memory_dos_impact.py --format json | python -m json.tool +``` + +## Interpreting Memory Results + +The **peak memory** metric shows the maximum memory allocated during unpickling: + +- **Without chunking**: Allocates full size immediately + - 10MB object → 10MB allocation upfront + +- **With chunking**: Allocates in 1MB chunks, grows geometrically + - 10MB object → starts with 1MB, grows: 2MB, 4MB, 8MB (final: ~10MB total) + - Peak is lower because allocation is incremental + +## Typical Results + +On a system with the PR applied, you should see: + +``` +1.00MiB Test Results + bytes_1.00MiB: ~0.3ms, 1.00MiB peak (just at threshold) + +2.00MiB Test Results + bytes_2.00MiB: ~0.8ms, 2.00MiB peak (chunked: 1MiB → 2MiB) + +10.00MiB Test Results + bytes_10.00MiB: ~3-5ms, 10.00MiB peak (chunked: 1→2→4→8→10 MiB) +``` + +Time overhead is minimal (~10-20% for very large objects), but memory safety is significantly improved. diff --git a/Tools/picklebench/memory_dos_impact.py b/Tools/picklebench/memory_dos_impact.py new file mode 100755 index 00000000000000..3bad6586c46943 --- /dev/null +++ b/Tools/picklebench/memory_dos_impact.py @@ -0,0 +1,1069 @@ +#!/usr/bin/env python3 +# +# Author: Claude Sonnet 4.5 as driven by gpshead +# +""" +Microbenchmark for pickle module chunked reading performance (GH PR #119204). + +This script generates Python data structures that act as antagonistic load +tests for the chunked reading code introduced to prevent memory exhaustion when +unpickling large objects. + +The PR adds chunked reading (1MB chunks) for: +- BINBYTES8 (large bytes) +- BINUNICODE8 (large strings) +- BYTEARRAY8 (large bytearrays) +- FRAME (large frames) +- LONG4 (large integers) + +Including an antagonistic mode that exercies memory denial of service pickles. + +Usage: + python memory_dos_impact.py --help +""" + +import argparse +import gc +import io +import json +import os +import pickle +import statistics +import struct +import subprocess +import sys +import tempfile +import tracemalloc +from pathlib import Path +from time import perf_counter +from typing import Any, Dict, List, Tuple, Optional + + +# Configuration +MIN_READ_BUF_SIZE = 1 << 20 # 1MB - matches pickle.py _MIN_READ_BUF_SIZE + +# Test sizes in MiB +DEFAULT_SIZES_MIB = [1, 2, 5, 10, 20, 50, 100, 200] + +# Convert to bytes, plus threshold boundary tests +DEFAULT_SIZES = ( + [999_000] # Below 1MiB (no chunking) + + [size * (1 << 20) for size in DEFAULT_SIZES_MIB] # MiB to bytes + + [1_048_577] # Just above 1MiB (minimal chunking overhead) +) +DEFAULT_SIZES.sort() + +# Baseline benchmark configuration +BASELINE_BENCHMARK_TIMEOUT_SECONDS = 600 # 10 minutes + +# Sparse memo attack test configuration +# Format: test_name -> (memo_index, baseline_memory_note) +SPARSE_MEMO_TESTS = { + "sparse_memo_1M": (1_000_000, "~8 MB array"), + "sparse_memo_100M": (100_000_000, "~800 MB array"), + "sparse_memo_1B": (1_000_000_000, "~8 GB array"), +} + + +# Utility functions + +def _extract_size_mb(size_key: str) -> float: + """Extract numeric MiB value from size_key like '10.00MB' or '1.00MiB'. + + Returns 0.0 for non-numeric keys (they'll be sorted last). + """ + try: + return float(size_key.replace('MB', '').replace('MiB', '')) + except ValueError: + return 999999.0 # Put non-numeric keys last + + +def _format_output(results: Dict[str, Dict[str, Any]], format_type: str, is_antagonistic: bool) -> str: + """Format benchmark results according to requested format. + + Args: + results: Benchmark results dictionary + format_type: Output format ('text', 'markdown', or 'json') + is_antagonistic: Whether these are antagonistic (DoS) test results + + Returns: + Formatted output string + """ + if format_type == 'json': + return Reporter.format_json(results) + elif is_antagonistic: + # Antagonistic mode uses specialized formatter for text/markdown + return Reporter.format_antagonistic(results) + elif format_type == 'text': + return Reporter.format_text(results) + elif format_type == 'markdown': + return Reporter.format_markdown(results) + else: + # Default to text format + return Reporter.format_text(results) + + +class AntagonisticGenerator: + """Generate malicious/truncated pickles for DoS protection testing. + + These pickles claim large sizes but provide minimal data, causing them to fail + during unpickling. They demonstrate the memory protection of chunked reading. + """ + + @staticmethod + def truncated_binbytes8(claimed_size: int, actual_size: int = 1024) -> bytes: + """BINBYTES8 claiming `claimed_size` but providing only `actual_size` bytes. + + This will fail with UnpicklingError but demonstrates peak memory usage. + Before PR: Allocates full claimed_size + After PR: Allocates in 1MB chunks, fails fast + """ + return b'\x8e' + struct.pack(' bytes: + """BINUNICODE8 claiming `claimed_size` but providing only `actual_size` bytes.""" + return b'\x8d' + struct.pack(' bytes: + """BYTEARRAY8 claiming `claimed_size` but providing only `actual_size` bytes.""" + return b'\x96' + struct.pack(' bytes: + """FRAME claiming `claimed_size` but providing minimal data.""" + return b'\x95' + struct.pack(' bytes: + """LONG_BINPUT with huge sparse index. + + Before PR: Tries to allocate array with `index` slots (OOM) + After PR: Uses dict-based memo for sparse indices + """ + return (b'(]r' + struct.pack(' bytes: + """Multiple BINBYTES8 claims in sequence. + + Tests that multiple large claims don't accumulate memory. + """ + data = b'(' # MARK + for _ in range(count): + data += b'\x8e' + struct.pack(' bytes: + """Generate random bytes of specified size.""" + return os.urandom(size) + + @staticmethod + def large_string_ascii(size: int) -> str: + """Generate ASCII string of specified size.""" + return 'x' * size + + @staticmethod + def large_string_multibyte(size: int) -> str: + """Generate multibyte UTF-8 string (3 bytes per char for €).""" + # Each € is 3 bytes in UTF-8 + return '€' * (size // 3) + + @staticmethod + def large_bytearray(size: int) -> bytearray: + """Generate bytearray of specified size.""" + return bytearray(os.urandom(size)) + + @staticmethod + def list_of_large_bytes(item_size: int, count: int) -> List[bytes]: + """Generate list containing multiple large bytes objects.""" + return [os.urandom(item_size) for _ in range(count)] + + @staticmethod + def dict_with_large_values(value_size: int, count: int) -> Dict[str, bytes]: + """Generate dict with large bytes values.""" + return { + f'key_{i}': os.urandom(value_size) + for i in range(count) + } + + @staticmethod + def nested_structure(size: int) -> Dict[str, Any]: + """Generate nested structure with various large objects.""" + chunk_size = size // 4 + return { + 'name': 'test_object', + 'data': { + 'bytes': os.urandom(chunk_size), + 'string': 's' * chunk_size, + 'bytearray': bytearray(b'b' * chunk_size), + }, + 'items': [os.urandom(chunk_size // 4) for _ in range(4)], + 'metadata': { + 'size': size, + 'type': 'nested', + }, + } + + @staticmethod + def tuple_of_large_objects(size: int) -> Tuple[bytes, str, bytearray]: + """Generate tuple with large objects (immutable, different pickle path).""" + chunk_size = size // 3 + return ( + os.urandom(chunk_size), + 'x' * chunk_size, + bytearray(b'y' * chunk_size), + ) + + +class PickleBenchmark: + """Benchmark pickle unpickling performance and memory usage.""" + + def __init__(self, obj: Any, protocol: int = 5, iterations: int = 3): + self.obj = obj + self.protocol = protocol + self.iterations = iterations + self.pickle_data = pickle.dumps(obj, protocol=protocol) + self.pickle_size = len(self.pickle_data) + + def benchmark_time(self) -> Dict[str, float]: + """Measure unpickling time over multiple iterations.""" + times = [] + + for _ in range(self.iterations): + start = perf_counter() + result = pickle.loads(self.pickle_data) + elapsed = perf_counter() - start + times.append(elapsed) + + # Verify correctness (first iteration only) + if len(times) == 1: + if result != self.obj: + raise ValueError("Unpickled object doesn't match original!") + + return { + 'mean': statistics.mean(times), + 'median': statistics.median(times), + 'stdev': statistics.stdev(times) if len(times) > 1 else 0.0, + 'min': min(times), + 'max': max(times), + } + + def benchmark_memory(self) -> int: + """Measure peak memory usage during unpickling.""" + tracemalloc.start() + + # Warmup + pickle.loads(self.pickle_data) + + # Actual measurement + gc.collect() + tracemalloc.reset_peak() + result = pickle.loads(self.pickle_data) + current, peak = tracemalloc.get_traced_memory() + + tracemalloc.stop() + + # Verify correctness + if result != self.obj: + raise ValueError("Unpickled object doesn't match original!") + + return peak + + def run_all(self) -> Dict[str, Any]: + """Run all benchmarks and return comprehensive results.""" + time_stats = self.benchmark_time() + peak_memory = self.benchmark_memory() + + return { + 'pickle_size_bytes': self.pickle_size, + 'pickle_size_mb': self.pickle_size / (1 << 20), + 'protocol': self.protocol, + 'time': time_stats, + 'memory_peak_bytes': peak_memory, + 'memory_peak_mb': peak_memory / (1 << 20), + 'iterations': self.iterations, + } + + +class AntagonisticBenchmark: + """Benchmark antagonistic/malicious pickles that demonstrate DoS protection. + + These pickles are designed to FAIL unpickling, but we measure peak memory + usage before the failure to demonstrate the memory protection. + """ + + def __init__(self, pickle_data: bytes, name: str): + self.pickle_data = pickle_data + self.name = name + + def measure_peak_memory(self, expect_success: bool = False) -> Dict[str, Any]: + """Measure peak memory when attempting to unpickle antagonistic data. + + Args: + expect_success: If True, test expects successful unpickling (e.g., sparse memo). + If False, test expects failure (e.g., truncated data). + """ + tracemalloc.start() + gc.collect() + tracemalloc.reset_peak() + + error_type = None + error_msg = None + succeeded = False + + try: + result = pickle.loads(self.pickle_data) + succeeded = True + if expect_success: + error_type = "Success (expected)" + else: + error_type = "WARNING: Expected failure but succeeded" + except (pickle.UnpicklingError, EOFError, ValueError, OverflowError) as e: + if expect_success: + error_type = f"UNEXPECTED FAILURE: {type(e).__name__}" + error_msg = str(e)[:100] + else: + # Expected failure for truncated data tests + error_type = type(e).__name__ + error_msg = str(e)[:100] + + current, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + + return { + 'test_name': self.name, + 'peak_memory_bytes': peak, + 'peak_memory_mb': peak / (1 << 20), + 'error_type': error_type, + 'error_msg': error_msg, + 'pickle_size_bytes': len(self.pickle_data), + 'expected_outcome': 'success' if expect_success else 'failure', + 'succeeded': succeeded, + } + + +class AntagonisticTestSuite: + """Manage a suite of antagonistic (DoS protection) tests.""" + + # Default sizes in MB to claim (will provide only 1KB actual data) + DEFAULT_ANTAGONISTIC_SIZES_MB = [10, 50, 100, 500, 1000, 5000] + + def __init__(self, claimed_sizes_mb: List[int]): + self.claimed_sizes_mb = claimed_sizes_mb + + def _run_truncated_test( + self, + test_type: str, + generator_func, + claimed_bytes: int, + claimed_mb: int, + size_key: str, + all_results: Dict[str, Dict[str, Any]] + ) -> None: + """Run a single truncated data test and store results. + + Args: + test_type: Type identifier (e.g., 'binbytes8', 'binunicode8') + generator_func: Function to generate malicious pickle data + claimed_bytes: Size claimed in the pickle (bytes) + claimed_mb: Size claimed in the pickle (MB) + size_key: Result key for this size (e.g., '10MB') + all_results: Dictionary to store results in + """ + test_name = f"{test_type}_{size_key}_claim" + data = generator_func(claimed_bytes) + bench = AntagonisticBenchmark(data, test_name) + result = bench.measure_peak_memory(expect_success=False) + result['claimed_mb'] = claimed_mb + all_results[size_key][test_name] = result + + def run_all_tests(self) -> Dict[str, Dict[str, Any]]: + """Run comprehensive antagonistic test suite.""" + all_results = {} + + for claimed_mb in self.claimed_sizes_mb: + claimed_bytes = claimed_mb << 20 + size_key = f"{claimed_mb}MB" + all_results[size_key] = {} + + # Run truncated data tests (expect failure) + self._run_truncated_test('binbytes8', AntagonisticGenerator.truncated_binbytes8, + claimed_bytes, claimed_mb, size_key, all_results) + self._run_truncated_test('binunicode8', AntagonisticGenerator.truncated_binunicode8, + claimed_bytes, claimed_mb, size_key, all_results) + self._run_truncated_test('bytearray8', AntagonisticGenerator.truncated_bytearray8, + claimed_bytes, claimed_mb, size_key, all_results) + self._run_truncated_test('frame', AntagonisticGenerator.truncated_frame, + claimed_bytes, claimed_mb, size_key, all_results) + + # Test 5: Sparse memo (expect success - dict-based memo works!) + all_results["Sparse Memo (Success Expected)"] = {} + for test_name, (index, baseline_note) in SPARSE_MEMO_TESTS.items(): + data = AntagonisticGenerator.sparse_memo_attack(index) + bench = AntagonisticBenchmark(data, test_name) + result = bench.measure_peak_memory(expect_success=True) + result['claimed_mb'] = "N/A" + result['baseline_note'] = f"Without PR: {baseline_note}" + all_results["Sparse Memo (Success Expected)"][test_name] = result + + # Test 6: Multi-claim attack (expect failure) + test_name = "multi_claim_10x100MB" + data = AntagonisticGenerator.multi_claim_attack(10, 100 << 20) + bench = AntagonisticBenchmark(data, test_name) + result = bench.measure_peak_memory(expect_success=False) + result['claimed_mb'] = 1000 # 10 * 100MB + all_results["Multi-Claim (Failure Expected)"] = {test_name: result} + + return all_results + + +class TestSuite: + """Manage a suite of benchmark tests.""" + + def __init__(self, sizes: List[int], protocol: int = 5, iterations: int = 3): + self.sizes = sizes + self.protocol = protocol + self.iterations = iterations + self.results = {} + + def run_test(self, name: str, obj: Any) -> Dict[str, Any]: + """Run benchmark for a single test object.""" + bench = PickleBenchmark(obj, self.protocol, self.iterations) + results = bench.run_all() + results['test_name'] = name + results['object_type'] = type(obj).__name__ + return results + + def run_all_tests(self) -> Dict[str, Dict[str, Any]]: + """Run comprehensive test suite across all sizes and types.""" + all_results = {} + + for size in self.sizes: + size_key = f"{size / (1 << 20):.2f}MB" + all_results[size_key] = {} + + # Test 1: Large bytes object (BINBYTES8) + test_name = f"bytes_{size_key}" + obj = DataGenerator.large_bytes(size) + all_results[size_key][test_name] = self.run_test(test_name, obj) + + # Test 2: Large ASCII string (BINUNICODE8) + test_name = f"string_ascii_{size_key}" + obj = DataGenerator.large_string_ascii(size) + all_results[size_key][test_name] = self.run_test(test_name, obj) + + # Test 3: Large multibyte UTF-8 string + if size >= 3: + test_name = f"string_utf8_{size_key}" + obj = DataGenerator.large_string_multibyte(size) + all_results[size_key][test_name] = self.run_test(test_name, obj) + + # Test 4: Large bytearray (BYTEARRAY8, protocol 5) + if self.protocol >= 5: + test_name = f"bytearray_{size_key}" + obj = DataGenerator.large_bytearray(size) + all_results[size_key][test_name] = self.run_test(test_name, obj) + + # Test 5: List of large objects (repeated chunking) + if size >= MIN_READ_BUF_SIZE * 2: + test_name = f"list_large_items_{size_key}" + item_size = size // 5 + obj = DataGenerator.list_of_large_bytes(item_size, 5) + all_results[size_key][test_name] = self.run_test(test_name, obj) + + # Test 6: Dict with large values + if size >= MIN_READ_BUF_SIZE * 2: + test_name = f"dict_large_values_{size_key}" + value_size = size // 3 + obj = DataGenerator.dict_with_large_values(value_size, 3) + all_results[size_key][test_name] = self.run_test(test_name, obj) + + # Test 7: Nested structure + if size >= MIN_READ_BUF_SIZE: + test_name = f"nested_{size_key}" + obj = DataGenerator.nested_structure(size) + all_results[size_key][test_name] = self.run_test(test_name, obj) + + # Test 8: Tuple (immutable) + if size >= 3: + test_name = f"tuple_{size_key}" + obj = DataGenerator.tuple_of_large_objects(size) + all_results[size_key][test_name] = self.run_test(test_name, obj) + + return all_results + + +class Comparator: + """Compare benchmark results between current and baseline interpreters.""" + + @staticmethod + def _extract_json_from_output(output: str) -> Dict[str, Dict[str, Any]]: + """Extract JSON data from subprocess output. + + Skips any print statements before the JSON output and parses the JSON. + + Args: + output: Raw stdout from subprocess + + Returns: + Parsed JSON as dictionary + + Raises: + SystemExit: If JSON cannot be found or parsed + """ + output_lines = output.strip().split('\n') + json_start = -1 + for i, line in enumerate(output_lines): + if line.strip().startswith('{'): + json_start = i + break + + if json_start == -1: + print("Error: Could not find JSON output from baseline", file=sys.stderr) + sys.exit(1) + + json_output = '\n'.join(output_lines[json_start:]) + try: + return json.loads(json_output) + except json.JSONDecodeError as e: + print(f"Error: Could not parse baseline JSON output: {e}", file=sys.stderr) + sys.exit(1) + + @staticmethod + def run_baseline_benchmark(baseline_python: str, args: argparse.Namespace) -> Dict[str, Dict[str, Any]]: + """Run the benchmark using the baseline Python interpreter.""" + # Build command to run this script with baseline Python + cmd = [ + baseline_python, + __file__, + '--format', 'json', + '--protocol', str(args.protocol), + '--iterations', str(args.iterations), + ] + + if args.sizes is not None: + cmd.extend(['--sizes'] + [str(s) for s in args.sizes]) + + if args.antagonistic: + cmd.append('--antagonistic') + + print(f"\nRunning baseline benchmark with: {baseline_python}") + print(f"Command: {' '.join(cmd)}\n") + + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=BASELINE_BENCHMARK_TIMEOUT_SECONDS, + ) + + if result.returncode != 0: + print(f"Error running baseline benchmark:", file=sys.stderr) + print(result.stderr, file=sys.stderr) + sys.exit(1) + + # Extract and parse JSON from output + return Comparator._extract_json_from_output(result.stdout) + + except subprocess.TimeoutExpired: + print("Error: Baseline benchmark timed out", file=sys.stderr) + sys.exit(1) + + @staticmethod + def calculate_change(baseline_value: float, current_value: float) -> float: + """Calculate percentage change from baseline to current.""" + if baseline_value == 0: + return 0.0 + return ((current_value - baseline_value) / baseline_value) * 100 + + @staticmethod + def format_comparison( + current_results: Dict[str, Dict[str, Any]], + baseline_results: Dict[str, Dict[str, Any]] + ) -> str: + """Format comparison results as readable text.""" + lines = [] + lines.append("=" * 100) + lines.append("Pickle Unpickling Benchmark Comparison") + lines.append("=" * 100) + lines.append("") + lines.append("Legend: Current vs Baseline | % Change (+ is slower/more memory, - is faster/less memory)") + lines.append("") + + # Sort size keys numerically + for size_key in sorted(current_results.keys(), key=_extract_size_mb): + if size_key not in baseline_results: + continue + + lines.append(f"\n{size_key} Comparison") + lines.append("-" * 100) + + current_tests = current_results[size_key] + baseline_tests = baseline_results[size_key] + + for test_name in sorted(current_tests.keys()): + if test_name not in baseline_tests: + continue + + curr = current_tests[test_name] + base = baseline_tests[test_name] + + time_change = Comparator.calculate_change( + base['time']['mean'], curr['time']['mean'] + ) + mem_change = Comparator.calculate_change( + base['memory_peak_mb'], curr['memory_peak_mb'] + ) + + lines.append(f"\n {curr['test_name']}") + lines.append(f" Time: {curr['time']['mean']*1000:6.2f}ms vs {base['time']['mean']*1000:6.2f}ms | " + f"{time_change:+6.1f}%") + lines.append(f" Memory: {curr['memory_peak_mb']:6.2f}MB vs {base['memory_peak_mb']:6.2f}MB | " + f"{mem_change:+6.1f}%") + + lines.append("\n" + "=" * 100) + lines.append("\nSummary:") + + # Calculate overall statistics + time_changes = [] + mem_changes = [] + + for size_key in current_results.keys(): + if size_key not in baseline_results: + continue + for test_name in current_results[size_key].keys(): + if test_name not in baseline_results[size_key]: + continue + curr = current_results[size_key][test_name] + base = baseline_results[size_key][test_name] + + time_changes.append(Comparator.calculate_change( + base['time']['mean'], curr['time']['mean'] + )) + mem_changes.append(Comparator.calculate_change( + base['memory_peak_mb'], curr['memory_peak_mb'] + )) + + if time_changes: + lines.append(f" Time change: mean={statistics.mean(time_changes):+.1f}%, " + f"median={statistics.median(time_changes):+.1f}%") + if mem_changes: + lines.append(f" Memory change: mean={statistics.mean(mem_changes):+.1f}%, " + f"median={statistics.median(mem_changes):+.1f}%") + + lines.append("=" * 100) + return "\n".join(lines) + + @staticmethod + def format_antagonistic_comparison( + current_results: Dict[str, Dict[str, Any]], + baseline_results: Dict[str, Dict[str, Any]] + ) -> str: + """Format antagonistic benchmark comparison results.""" + lines = [] + lines.append("=" * 100) + lines.append("Antagonistic Pickle Benchmark Comparison (Memory DoS Protection)") + lines.append("=" * 100) + lines.append("") + lines.append("Legend: Current vs Baseline | Memory Change (- is better, shows memory saved)") + lines.append("") + lines.append("This compares TWO types of DoS protection:") + lines.append(" 1. Truncated data → Baseline allocates full claimed size, Current uses chunked reading") + lines.append(" 2. Sparse memo → Baseline uses huge arrays, Current uses dict-based memo") + lines.append("") + + # Track statistics + truncated_memory_changes = [] + sparse_memory_changes = [] + + # Sort size keys numerically + for size_key in sorted(current_results.keys(), key=_extract_size_mb): + if size_key not in baseline_results: + continue + + lines.append(f"\n{size_key} Comparison") + lines.append("-" * 100) + + current_tests = current_results[size_key] + baseline_tests = baseline_results[size_key] + + for test_name in sorted(current_tests.keys()): + if test_name not in baseline_tests: + continue + + curr = current_tests[test_name] + base = baseline_tests[test_name] + + curr_peak_mb = curr['peak_memory_mb'] + base_peak_mb = base['peak_memory_mb'] + expected_outcome = curr.get('expected_outcome', 'failure') + + mem_change = Comparator.calculate_change(base_peak_mb, curr_peak_mb) + mem_saved_mb = base_peak_mb - curr_peak_mb + + lines.append(f"\n {curr['test_name']}") + lines.append(f" Memory: {curr_peak_mb:6.2f}MB vs {base_peak_mb:6.2f}MB | " + f"{mem_change:+6.1f}% ({mem_saved_mb:+.2f}MB saved)") + + # Track based on test type + if expected_outcome == 'success': + sparse_memory_changes.append(mem_change) + if curr.get('baseline_note'): + lines.append(f" Note: {curr['baseline_note']}") + else: + truncated_memory_changes.append(mem_change) + claimed_mb = curr.get('claimed_mb', 'N/A') + if claimed_mb != 'N/A': + lines.append(f" Claimed: {claimed_mb:,}MB") + + # Show status + curr_status = curr.get('error_type', 'Unknown') + base_status = base.get('error_type', 'Unknown') + if curr_status != base_status: + lines.append(f" Status: {curr_status} (baseline: {base_status})") + else: + lines.append(f" Status: {curr_status}") + + lines.append("\n" + "=" * 100) + lines.append("\nSummary:") + lines.append("") + + if truncated_memory_changes: + lines.append(" Truncated Data Protection (chunked reading):") + lines.append(f" Mean memory change: {statistics.mean(truncated_memory_changes):+.1f}%") + lines.append(f" Median memory change: {statistics.median(truncated_memory_changes):+.1f}%") + avg_change = statistics.mean(truncated_memory_changes) + if avg_change < -50: + lines.append(f" Result: ✓ Dramatic memory reduction ({avg_change:.1f}%) - DoS protection working!") + elif avg_change < 0: + lines.append(f" Result: ✓ Memory reduced ({avg_change:.1f}%)") + else: + lines.append(f" Result: ⚠ Memory increased ({avg_change:.1f}%) - unexpected!") + lines.append("") + + if sparse_memory_changes: + lines.append(" Sparse Memo Protection (dict-based memo):") + lines.append(f" Mean memory change: {statistics.mean(sparse_memory_changes):+.1f}%") + lines.append(f" Median memory change: {statistics.median(sparse_memory_changes):+.1f}%") + avg_change = statistics.mean(sparse_memory_changes) + if avg_change < -50: + lines.append(f" Result: ✓ Dramatic memory reduction ({avg_change:.1f}%) - Dict optimization working!") + elif avg_change < 0: + lines.append(f" Result: ✓ Memory reduced ({avg_change:.1f}%)") + else: + lines.append(f" Result: ⚠ Memory increased ({avg_change:.1f}%) - unexpected!") + + lines.append("") + lines.append("=" * 100) + return "\n".join(lines) + + +class Reporter: + """Format and display benchmark results.""" + + @staticmethod + def format_text(results: Dict[str, Dict[str, Any]]) -> str: + """Format results as readable text.""" + lines = [] + lines.append("=" * 80) + lines.append("Pickle Unpickling Benchmark Results") + lines.append("=" * 80) + lines.append("") + + for size_key, tests in results.items(): + lines.append(f"\n{size_key} Test Results") + lines.append("-" * 80) + + for test_name, data in tests.items(): + lines.append(f"\n Test: {data['test_name']}") + lines.append(f" Type: {data['object_type']}") + lines.append(f" Pickle size: {data['pickle_size_mb']:.2f} MB") + lines.append(f" Time (mean): {data['time']['mean']*1000:.2f} ms") + lines.append(f" Time (stdev): {data['time']['stdev']*1000:.2f} ms") + lines.append(f" Peak memory: {data['memory_peak_mb']:.2f} MB") + lines.append(f" Protocol: {data['protocol']}") + + lines.append("\n" + "=" * 80) + return "\n".join(lines) + + @staticmethod + def format_markdown(results: Dict[str, Dict[str, Any]]) -> str: + """Format results as markdown table.""" + lines = [] + lines.append("# Pickle Unpickling Benchmark Results\n") + + for size_key, tests in results.items(): + lines.append(f"## {size_key}\n") + lines.append("| Test | Type | Pickle Size (MB) | Time (ms) | Stdev (ms) | Peak Memory (MB) |") + lines.append("|------|------|------------------|-----------|------------|------------------|") + + for test_name, data in tests.items(): + lines.append( + f"| {data['test_name']} | " + f"{data['object_type']} | " + f"{data['pickle_size_mb']:.2f} | " + f"{data['time']['mean']*1000:.2f} | " + f"{data['time']['stdev']*1000:.2f} | " + f"{data['memory_peak_mb']:.2f} |" + ) + lines.append("") + + return "\n".join(lines) + + @staticmethod + def format_json(results: Dict[str, Dict[str, Any]]) -> str: + """Format results as JSON.""" + import json + return json.dumps(results, indent=2) + + @staticmethod + def format_antagonistic(results: Dict[str, Dict[str, Any]]) -> str: + """Format antagonistic benchmark results.""" + lines = [] + lines.append("=" * 100) + lines.append("Antagonistic Pickle Benchmark (Memory DoS Protection Test)") + lines.append("=" * 100) + lines.append("") + lines.append("This benchmark tests TWO types of DoS protection:") + lines.append(" 1. Truncated data attacks → Expect FAILURE with minimal memory before failure") + lines.append(" 2. Sparse memo attacks → Expect SUCCESS with dict-based memo (vs huge array)") + lines.append("") + + # Sort size keys numerically + for size_key in sorted(results.keys(), key=_extract_size_mb): + tests = results[size_key] + + # Determine test type from first test + if tests: + first_test = next(iter(tests.values())) + expected_outcome = first_test.get('expected_outcome', 'failure') + claimed_mb = first_test.get('claimed_mb', 'N/A') + + # Header varies by test type + if "Sparse Memo" in size_key: + lines.append(f"\n{size_key}") + lines.append("-" * 100) + elif "Multi-Claim" in size_key: + lines.append(f"\n{size_key}") + lines.append("-" * 100) + elif claimed_mb != 'N/A': + lines.append(f"\n{size_key} Claimed (actual: 1KB) - Expect Failure") + lines.append("-" * 100) + else: + lines.append(f"\n{size_key}") + lines.append("-" * 100) + + for test_name, data in tests.items(): + peak_mb = data['peak_memory_mb'] + claimed = data.get('claimed_mb', 'N/A') + expected_outcome = data.get('expected_outcome', 'failure') + succeeded = data.get('succeeded', False) + baseline_note = data.get('baseline_note', '') + + lines.append(f" {data['test_name']}") + + # Format output based on test type + if expected_outcome == 'success': + # Sparse memo test - show success with dict + status_icon = "✓" if succeeded else "✗" + lines.append(f" Peak memory: {peak_mb:8.2f} MB {status_icon}") + lines.append(f" Status: {data['error_type']}") + if baseline_note: + lines.append(f" {baseline_note}") + else: + # Truncated data test - show savings before failure + if claimed != 'N/A': + saved_mb = claimed - peak_mb + savings_pct = (saved_mb / claimed * 100) if claimed > 0 else 0 + lines.append(f" Peak memory: {peak_mb:8.2f} MB (claimed: {claimed:,} MB, saved: {saved_mb:.2f} MB, {savings_pct:.1f}%)") + else: + lines.append(f" Peak memory: {peak_mb:8.2f} MB") + lines.append(f" Status: {data['error_type']}") + + lines.append("\n" + "=" * 100) + + # Calculate statistics by test type + truncated_claimed = 0 + truncated_peak = 0 + truncated_count = 0 + + sparse_peak_total = 0 + sparse_count = 0 + + for size_key, tests in results.items(): + for test_name, data in tests.items(): + expected_outcome = data.get('expected_outcome', 'failure') + + if expected_outcome == 'failure': + # Truncated data test + claimed = data.get('claimed_mb', 0) + if claimed != 'N/A' and claimed > 0: + truncated_claimed += claimed + truncated_peak += data['peak_memory_mb'] + truncated_count += 1 + else: + # Sparse memo test + sparse_peak_total += data['peak_memory_mb'] + sparse_count += 1 + + lines.append("\nSummary:") + lines.append("") + + if truncated_count > 0: + avg_claimed = truncated_claimed / truncated_count + avg_peak = truncated_peak / truncated_count + avg_saved = avg_claimed - avg_peak + avg_savings_pct = (avg_saved / avg_claimed * 100) if avg_claimed > 0 else 0 + + lines.append(" Truncated Data Protection (chunked reading):") + lines.append(f" Average claimed: {avg_claimed:,.1f} MB") + lines.append(f" Average peak: {avg_peak:,.2f} MB") + lines.append(f" Average saved: {avg_saved:,.2f} MB ({avg_savings_pct:.1f}% reduction)") + lines.append(f" Status: ✓ Fails fast with minimal memory") + lines.append("") + + if sparse_count > 0: + avg_sparse_peak = sparse_peak_total / sparse_count + lines.append(" Sparse Memo Protection (dict-based memo):") + lines.append(f" Average peak: {avg_sparse_peak:,.2f} MB") + lines.append(f" Status: ✓ Succeeds with dict (vs GB-sized arrays without PR)") + lines.append(f" Note: Compare with --baseline to see actual memory savings") + + lines.append("") + lines.append("=" * 100) + return "\n".join(lines) + + +def main(): + parser = argparse.ArgumentParser( + description="Benchmark pickle unpickling performance for large objects" + ) + parser.add_argument( + '--sizes', + type=int, + nargs='+', + default=None, + metavar='MiB', + help=f'Object sizes to test in MiB (default: {DEFAULT_SIZES_MIB})' + ) + parser.add_argument( + '--protocol', + type=int, + default=5, + choices=[0, 1, 2, 3, 4, 5], + help='Pickle protocol version (default: 5)' + ) + parser.add_argument( + '--iterations', + type=int, + default=3, + help='Number of benchmark iterations (default: 3)' + ) + parser.add_argument( + '--format', + choices=['text', 'markdown', 'json'], + default='text', + help='Output format (default: text)' + ) + parser.add_argument( + '--baseline', + type=str, + metavar='PYTHON', + help='Path to baseline Python interpreter for comparison (e.g., ../main-build/python)' + ) + parser.add_argument( + '--antagonistic', + action='store_true', + help='Run antagonistic/malicious pickle tests (DoS protection benchmark)' + ) + + args = parser.parse_args() + + # Handle antagonistic mode + if args.antagonistic: + # Antagonistic mode uses claimed sizes in MB, not actual data sizes + if args.sizes is None: + claimed_sizes_mb = AntagonisticTestSuite.DEFAULT_ANTAGONISTIC_SIZES_MB + else: + claimed_sizes_mb = args.sizes + + print(f"Running ANTAGONISTIC pickle benchmark (DoS protection test)...") + print(f"Claimed sizes: {claimed_sizes_mb} MiB (actual data: 1KB each)") + print(f"NOTE: These pickles will FAIL to unpickle (expected)") + print() + + # Run antagonistic benchmark suite + suite = AntagonisticTestSuite(claimed_sizes_mb) + results = suite.run_all_tests() + + # Format and display results + if args.baseline: + # Verify baseline Python exists + baseline_path = Path(args.baseline) + if not baseline_path.exists(): + print(f"Error: Baseline Python not found: {args.baseline}", file=sys.stderr) + return 1 + + # Run baseline benchmark + baseline_results = Comparator.run_baseline_benchmark(args.baseline, args) + + # Show comparison + comparison_output = Comparator.format_antagonistic_comparison(results, baseline_results) + print(comparison_output) + else: + # Format and display results + output = _format_output(results, args.format, is_antagonistic=True) + print(output) + + else: + # Normal mode: legitimate pickle benchmarks + # Convert sizes from MiB to bytes + if args.sizes is None: + sizes_bytes = DEFAULT_SIZES + else: + sizes_bytes = [size * (1 << 20) for size in args.sizes] + + print(f"Running pickle benchmark with protocol {args.protocol}...") + print(f"Test sizes: {[f'{s/(1<<20):.2f}MiB' for s in sizes_bytes]}") + print(f"Iterations per test: {args.iterations}") + print() + + # Run benchmark suite + suite = TestSuite(sizes_bytes, args.protocol, args.iterations) + results = suite.run_all_tests() + + # If baseline comparison requested, run baseline and compare + if args.baseline: + # Verify baseline Python exists + baseline_path = Path(args.baseline) + if not baseline_path.exists(): + print(f"Error: Baseline Python not found: {args.baseline}", file=sys.stderr) + return 1 + + # Run baseline benchmark + baseline_results = Comparator.run_baseline_benchmark(args.baseline, args) + + # Show comparison + comparison_output = Comparator.format_comparison(results, baseline_results) + print(comparison_output) + + else: + # Format and display results + output = _format_output(results, args.format, is_antagonistic=False) + print(output) + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) From dcac498e501db3983ce22851c88f88561ae46351 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Fri, 5 Dec 2025 19:36:28 +0000 Subject: [PATCH 269/638] gh-142318: Fix typing `'q'` at interactive help screen exiting Tachyon (#142319) --- Lib/profiling/sampling/live_collector/collector.py | 4 +++- .../test_live_collector_interaction.py | 13 +++++++++++++ .../2025-12-05-18-25-29.gh-issue-142318.EzcQ3N.rst | 2 ++ 3 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-12-05-18-25-29.gh-issue-142318.EzcQ3N.rst diff --git a/Lib/profiling/sampling/live_collector/collector.py b/Lib/profiling/sampling/live_collector/collector.py index 4b69275a2f077f..7adbf1bbe7f625 100644 --- a/Lib/profiling/sampling/live_collector/collector.py +++ b/Lib/profiling/sampling/live_collector/collector.py @@ -861,10 +861,12 @@ def _handle_input(self): # Handle help toggle keys if ch == ord("h") or ch == ord("H") or ch == ord("?"): self.show_help = not self.show_help + return # If showing help, any other key closes it - elif self.show_help and ch != -1: + if self.show_help and ch != -1: self.show_help = False + return # Handle regular commands if ch == ord("q") or ch == ord("Q"): diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_interaction.py b/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_interaction.py index 388f462cf21b3d..a5870366552854 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_interaction.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_interaction.py @@ -173,6 +173,19 @@ def test_help_with_question_mark(self): self.assertTrue(self.collector.show_help) + def test_help_dismiss_with_q_does_not_quit(self): + """Test that pressing 'q' while help is shown only closes help, not quit""" + self.assertFalse(self.collector.show_help) + self.display.simulate_input(ord("h")) + self.collector._handle_input() + self.assertTrue(self.collector.show_help) + + self.display.simulate_input(ord("q")) + self.collector._handle_input() + + self.assertFalse(self.collector.show_help) + self.assertTrue(self.collector.running) + def test_filter_clear(self): """Test clearing filter.""" self.collector.filter_pattern = "test" diff --git a/Misc/NEWS.d/next/Library/2025-12-05-18-25-29.gh-issue-142318.EzcQ3N.rst b/Misc/NEWS.d/next/Library/2025-12-05-18-25-29.gh-issue-142318.EzcQ3N.rst new file mode 100644 index 00000000000000..8710ebfb1a1a0a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-05-18-25-29.gh-issue-142318.EzcQ3N.rst @@ -0,0 +1,2 @@ +Fix typing ``'q'`` at the help of the interactive tachyon profiler exiting +the profiler. From 58e1c7a16f0926b1047c336eeed2849d5fff7c70 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 5 Dec 2025 13:35:50 -0800 Subject: [PATCH 270/638] Introduce `build-python` and `build-host` subcommands for `Tools/wasm/wasi` (GH-142266) It should make it easier when you need to rebuild just the e.g. host Python, but it requires ./configure to run. Co-authored-by: Emma Smith --- Tools/wasm/wasi/__main__.py | 56 ++++++++++++++++++++++++++----------- 1 file changed, 39 insertions(+), 17 deletions(-) diff --git a/Tools/wasm/wasi/__main__.py b/Tools/wasm/wasi/__main__.py index d95cc99c8ea28b..7e33ff4b477b3e 100644 --- a/Tools/wasm/wasi/__main__.py +++ b/Tools/wasm/wasi/__main__.py @@ -386,18 +386,6 @@ def make_wasi_python(context, working_dir): ) -def build_all(context): - """Build everything.""" - steps = [ - configure_build_python, - make_build_python, - configure_wasi_python, - make_wasi_python, - ] - for step in steps: - step(context) - - def clean_contents(context): """Delete all files created by this script.""" if CROSS_BUILD_DIR.exists(): @@ -409,6 +397,16 @@ def clean_contents(context): log("🧹", f"Deleting generated {LOCAL_SETUP} ...") +def build_steps(*steps): + """Construct a command from other steps.""" + + def builder(context): + for step in steps: + step(context) + + return builder + + def main(): default_host_triple = "wasm32-wasip1" default_wasi_sdk = find_wasi_sdk() @@ -438,6 +436,9 @@ def main(): make_build = subcommands.add_parser( "make-build-python", help="Run `make` for the build Python" ) + build_python = subcommands.add_parser( + "build-python", help="Build the build Python" + ) configure_host = subcommands.add_parser( "configure-host", help="Run `configure` for the " @@ -448,6 +449,9 @@ def main(): make_host = subcommands.add_parser( "make-host", help="Run `make` for the host/WASI" ) + build_host = subcommands.add_parser( + "build-host", help="Build the host/WASI Python" + ) subcommands.add_parser( "clean", help="Delete files and directories created by this script" ) @@ -455,8 +459,10 @@ def main(): build, configure_build, make_build, + build_python, configure_host, make_host, + build_host, ): subcommand.add_argument( "--quiet", @@ -471,7 +477,12 @@ def main(): default=default_logdir, help=f"Directory to store log files; defaults to {default_logdir}", ) - for subcommand in configure_build, configure_host: + for subcommand in ( + configure_build, + configure_host, + build_python, + build_host, + ): subcommand.add_argument( "--clean", action="store_true", @@ -479,11 +490,17 @@ def main(): dest="clean", help="Delete any relevant directories before building", ) - for subcommand in build, configure_build, configure_host: + for subcommand in ( + build, + configure_build, + configure_host, + build_python, + build_host, + ): subcommand.add_argument( "args", nargs="*", help="Extra arguments to pass to `configure`" ) - for subcommand in build, configure_host: + for subcommand in build, configure_host, build_host: subcommand.add_argument( "--wasi-sdk", type=pathlib.Path, @@ -499,7 +516,7 @@ def main(): help="Command template for running the WASI host; defaults to " f"`{default_host_runner}`", ) - for subcommand in build, configure_host, make_host: + for subcommand in build, configure_host, make_host, build_host: subcommand.add_argument( "--host-triple", action="store", @@ -511,12 +528,17 @@ def main(): context = parser.parse_args() context.init_dir = pathlib.Path().absolute() + build_build_python = build_steps(configure_build_python, make_build_python) + build_wasi_python = build_steps(configure_wasi_python, make_wasi_python) + dispatch = { "configure-build-python": configure_build_python, "make-build-python": make_build_python, + "build-python": build_build_python, "configure-host": configure_wasi_python, "make-host": make_wasi_python, - "build": build_all, + "build-host": build_wasi_python, + "build": build_steps(build_build_python, build_wasi_python), "clean": clean_contents, } dispatch[context.subcommand](context) From d49e6f38a7a0ca666df2c81329291291f0389682 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 5 Dec 2025 14:31:30 -0800 Subject: [PATCH 271/638] Extract data from `Tools/wasm/wasi` that varies between Python versions into a config file (GH-142273) This should allow for easier backporting of code. --- Tools/wasm/wasi/__main__.py | 18 ++++++++++++------ Tools/wasm/wasi/config.toml | 6 ++++++ 2 files changed, 18 insertions(+), 6 deletions(-) create mode 100644 Tools/wasm/wasi/config.toml diff --git a/Tools/wasm/wasi/__main__.py b/Tools/wasm/wasi/__main__.py index 7e33ff4b477b3e..3d9a2472d4dcbf 100644 --- a/Tools/wasm/wasi/__main__.py +++ b/Tools/wasm/wasi/__main__.py @@ -5,6 +5,8 @@ import functools import os +import tomllib + try: from os import process_cpu_count as cpu_count except ImportError: @@ -18,6 +20,7 @@ HERE = pathlib.Path(__file__).parent +# Path is: cpython/Tools/wasm/wasi CHECKOUT = HERE.parent.parent.parent assert (CHECKOUT / "configure").is_file(), ( "Please update the location of the file" @@ -213,9 +216,10 @@ def make_build_python(context, working_dir): log("🎉", f"{binary} {version}") -def find_wasi_sdk(): +def find_wasi_sdk(config): """Find the path to the WASI SDK.""" wasi_sdk_path = None + wasi_sdk_version = config["targets"]["wasi-sdk"] if wasi_sdk_path_env_var := os.environ.get("WASI_SDK_PATH"): wasi_sdk_path = pathlib.Path(wasi_sdk_path_env_var) @@ -229,7 +233,7 @@ def find_wasi_sdk(): # ``wasi-sdk-{WASI_SDK_VERSION}.0-x86_64-linux``. potential_sdks = [ path - for path in opt_path.glob(f"wasi-sdk-{WASI_SDK_VERSION}.0*") + for path in opt_path.glob(f"wasi-sdk-{wasi_sdk_version}.0*") if path.is_dir() ] if len(potential_sdks) == 1: @@ -245,12 +249,12 @@ def find_wasi_sdk(): found_version = version_details.splitlines()[0] # Make sure there's a trailing dot to avoid false positives if somehow the # supported version is a prefix of the found version (e.g. `25` and `2567`). - if not found_version.startswith(f"{WASI_SDK_VERSION}."): + if not found_version.startswith(f"{wasi_sdk_version}."): major_version = found_version.partition(".")[0] log( "⚠️", f" Found WASI SDK {major_version}, " - f"but WASI SDK {WASI_SDK_VERSION} is the supported version", + f"but WASI SDK {wasi_sdk_version} is the supported version", ) return wasi_sdk_path @@ -408,8 +412,10 @@ def builder(context): def main(): - default_host_triple = "wasm32-wasip1" - default_wasi_sdk = find_wasi_sdk() + with (HERE / "config.toml").open("rb") as file: + config = tomllib.load(file) + default_wasi_sdk = find_wasi_sdk(config) + default_host_triple = config["targets"]["host-triple"] default_host_runner = ( f"{WASMTIME_HOST_RUNNER_VAR} run " # For setting PYTHONPATH to the sysconfig data directory. diff --git a/Tools/wasm/wasi/config.toml b/Tools/wasm/wasi/config.toml new file mode 100644 index 00000000000000..7ca2f76f56dc7a --- /dev/null +++ b/Tools/wasm/wasi/config.toml @@ -0,0 +1,6 @@ +# Any data that can vary between Python versions is to be kept in this file. +# This allows for blanket copying of the WASI build code between supported +# Python versions. +[targets] +wasi-sdk = 29 +host-triple = "wasm32-wasip1" From eba449a1989265a923174142dd67dee074f90967 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 5 Dec 2025 15:27:16 -0800 Subject: [PATCH 272/638] GH-142234: Allow `--enable-wasm-dynamic-linking` under WASI (GH-142235) While CPython doesn't support `--enable-wasm-dynamic-linking`, external tools like componentize-py do and they have to patch around it. Since the flag is off by default, allowing the flag so external users can add/inject dynamic linking support seems acceptable. --- .../Build/2025-12-03-10-44-42.gh-issue-142234.i1kaFb.rst | 3 +++ configure | 5 +++-- configure.ac | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2025-12-03-10-44-42.gh-issue-142234.i1kaFb.rst diff --git a/Misc/NEWS.d/next/Build/2025-12-03-10-44-42.gh-issue-142234.i1kaFb.rst b/Misc/NEWS.d/next/Build/2025-12-03-10-44-42.gh-issue-142234.i1kaFb.rst new file mode 100644 index 00000000000000..a586512fc0a80b --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-12-03-10-44-42.gh-issue-142234.i1kaFb.rst @@ -0,0 +1,3 @@ +Allow ``--enable-wasm-dynamic-linking`` for WASI. While CPython doesn't +directly support it so external/downstream users do not have to patch in +support for the flag. diff --git a/configure b/configure index 1561f7f4134ac2..7561fb9c7ad90e 100755 --- a/configure +++ b/configure @@ -1824,7 +1824,8 @@ Optional Features: no) --enable-wasm-dynamic-linking Enable dynamic linking support for WebAssembly - (default is no) + (default is no); WASI requires an external dynamic + loader to handle imports --enable-wasm-pthreads Enable pthread emulation for WebAssembly (default is no) --enable-shared enable building a shared Python library (default is @@ -7415,7 +7416,7 @@ then : Emscripten) : ;; #( WASI) : - as_fn_error $? "WASI dynamic linking is not implemented yet." "$LINENO" 5 ;; #( + ;; #( *) : as_fn_error $? "--enable-wasm-dynamic-linking only applies to Emscripten and WASI" "$LINENO" 5 ;; diff --git a/configure.ac b/configure.ac index f2a7319d22d24b..fa24bc78a2645a 100644 --- a/configure.ac +++ b/configure.ac @@ -1323,11 +1323,11 @@ dnl See https://emscripten.org/docs/compiling/Dynamic-Linking.html AC_MSG_CHECKING([for --enable-wasm-dynamic-linking]) AC_ARG_ENABLE([wasm-dynamic-linking], [AS_HELP_STRING([--enable-wasm-dynamic-linking], - [Enable dynamic linking support for WebAssembly (default is no)])], + [Enable dynamic linking support for WebAssembly (default is no); WASI requires an external dynamic loader to handle imports])], [ AS_CASE([$ac_sys_system], [Emscripten], [], - [WASI], [AC_MSG_ERROR([WASI dynamic linking is not implemented yet.])], + [WASI], [], [AC_MSG_ERROR([--enable-wasm-dynamic-linking only applies to Emscripten and WASI])] ) ], [ From d1194439363a6896069c5c3cf66e95a14f79bc00 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 6 Dec 2025 12:27:31 +0100 Subject: [PATCH 273/638] Remove unused imports (#142320) --- Lib/cProfile.py | 1 - Lib/profiling/sampling/heatmap_collector.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/cProfile.py b/Lib/cProfile.py index 4af82f2cb8c848..cc6255f61ae211 100644 --- a/Lib/cProfile.py +++ b/Lib/cProfile.py @@ -9,6 +9,5 @@ __all__ = ["run", "runctx", "Profile"] if __name__ == "__main__": - import sys from profiling.tracing.__main__ import main main() diff --git a/Lib/profiling/sampling/heatmap_collector.py b/Lib/profiling/sampling/heatmap_collector.py index eb51ce33b28a52..eb128aba9b197f 100644 --- a/Lib/profiling/sampling/heatmap_collector.py +++ b/Lib/profiling/sampling/heatmap_collector.py @@ -11,7 +11,7 @@ import sys from dataclasses import dataclass, field from pathlib import Path -from typing import Dict, List, Tuple, Optional, Any +from typing import Dict, List, Tuple from ._css_utils import get_combined_css from .stack_collector import StackTraceCollector From 61823a5382e8c0c0292e90a46ae3e1859b7f278b Mon Sep 17 00:00:00 2001 From: "Y. Z. Chen" <754097987@qq.com> Date: Sat, 6 Dec 2025 21:05:20 +0800 Subject: [PATCH 274/638] Docs: fix RFC index reference for TLS 1.3 (#142262) --- Lib/ssl.py | 2 +- Lib/test/test_ssl.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/ssl.py b/Lib/ssl.py index 7ad7969a8217f8..67a2990b2817e2 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -185,7 +185,7 @@ class _TLSContentType: class _TLSAlertType: """Alert types for TLSContentType.ALERT messages - See RFC 8466, section B.2 + See RFC 8446, section B.2 """ CLOSE_NOTIFY = 0 UNEXPECTED_MESSAGE = 10 diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 09de32f8371ae9..ebdf5455163c65 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -5613,7 +5613,7 @@ def test_tlsalerttype(self): class Checked_TLSAlertType(enum.IntEnum): """Alert types for TLSContentType.ALERT messages - See RFC 8466, section B.2 + See RFC 8446, section B.2 """ CLOSE_NOTIFY = 0 UNEXPECTED_MESSAGE = 10 From 5be3405e4e94e494f3f2c4507d8c32c2c04bb2ee Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Sat, 6 Dec 2025 07:12:21 -0800 Subject: [PATCH 275/638] GH-75949: Fix argparse dropping '|' in mutually exclusive groups on line wrap (#142312) --- Lib/argparse.py | 33 ++++++++++++++++--- Lib/test/test_argparse.py | 19 +++++++++++ ...5-12-05-16-39-17.gh-issue-75949.pHxW98.rst | 1 + 3 files changed, 49 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-12-05-16-39-17.gh-issue-75949.pHxW98.rst diff --git a/Lib/argparse.py b/Lib/argparse.py index 10393b6a02b0be..07d7d77e8845be 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -353,8 +353,14 @@ def _format_usage(self, usage, actions, groups, prefix): if len(prefix) + len(self._decolor(usage)) > text_width: # break usage into wrappable parts - opt_parts = self._get_actions_usage_parts(optionals, groups) - pos_parts = self._get_actions_usage_parts(positionals, groups) + # keep optionals and positionals together to preserve + # mutually exclusive group formatting (gh-75949) + all_actions = optionals + positionals + parts, pos_start = self._get_actions_usage_parts_with_split( + all_actions, groups, len(optionals) + ) + opt_parts = parts[:pos_start] + pos_parts = parts[pos_start:] # helper for wrapping lines def get_lines(parts, indent, prefix=None): @@ -418,6 +424,17 @@ def _is_long_option(self, string): return len(string) > 2 def _get_actions_usage_parts(self, actions, groups): + parts, _ = self._get_actions_usage_parts_with_split(actions, groups) + return parts + + def _get_actions_usage_parts_with_split(self, actions, groups, opt_count=None): + """Get usage parts with split index for optionals/positionals. + + Returns (parts, pos_start) where pos_start is the index in parts + where positionals begin. When opt_count is None, pos_start is None. + This preserves mutually exclusive group formatting across the + optionals/positionals boundary (gh-75949). + """ # find group indices and identify actions in groups group_actions = set() inserts = {} @@ -513,8 +530,16 @@ def _get_actions_usage_parts(self, actions, groups): for i in range(start + group_size, end): parts[i] = None - # return the usage parts - return [item for item in parts if item is not None] + # if opt_count is provided, calculate where positionals start in + # the final parts list (for wrapping onto separate lines). + # Count before filtering None entries since indices shift after. + if opt_count is not None: + pos_start = sum(1 for p in parts[:opt_count] if p is not None) + else: + pos_start = None + + # return the usage parts and split point (gh-75949) + return [item for item in parts if item is not None], pos_start def _format_text(self, text): if '%(prog)' in text: diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index ef90d4bcbb2a36..dff7ba750fa559 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -4966,6 +4966,25 @@ def test_long_mutex_groups_wrap(self): ''') self.assertEqual(parser.format_usage(), usage) + def test_mutex_groups_with_mixed_optionals_positionals_wrap(self): + # https://github.com/python/cpython/issues/75949 + # Mutually exclusive groups containing both optionals and positionals + # should preserve pipe separators when the usage line wraps. + parser = argparse.ArgumentParser(prog='PROG') + g = parser.add_mutually_exclusive_group() + g.add_argument('-v', '--verbose', action='store_true') + g.add_argument('-q', '--quiet', action='store_true') + g.add_argument('-x', '--extra-long-option-name', nargs='?') + g.add_argument('-y', '--yet-another-long-option', nargs='?') + g.add_argument('positional', nargs='?') + + usage = textwrap.dedent('''\ + usage: PROG [-h] [-v | -q | -x [EXTRA_LONG_OPTION_NAME] | + -y [YET_ANOTHER_LONG_OPTION] | + positional] + ''') + self.assertEqual(parser.format_usage(), usage) + class TestHelpVariableExpansion(HelpTestCase): """Test that variables are expanded properly in help messages""" diff --git a/Misc/NEWS.d/next/Library/2025-12-05-16-39-17.gh-issue-75949.pHxW98.rst b/Misc/NEWS.d/next/Library/2025-12-05-16-39-17.gh-issue-75949.pHxW98.rst new file mode 100644 index 00000000000000..5ca3fc05b9816d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-05-16-39-17.gh-issue-75949.pHxW98.rst @@ -0,0 +1 @@ +Fix :mod:`argparse` to preserve ``|`` separators in mutually exclusive groups when the usage line wraps due to length. From 70c27ce94b2c18f375c10e508e7d9323ae795496 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 6 Dec 2025 20:03:45 +0200 Subject: [PATCH 276/638] gh-142332: Fix usage formatting for positional arguments in mutually exclusive groups in argparse (GH-142333) --- Lib/argparse.py | 8 ++----- Lib/test/test_argparse.py | 23 ++++++++++++++++++- ...-12-06-13-02-13.gh-issue-142332.PNvXCV.rst | 2 ++ 3 files changed, 26 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-12-06-13-02-13.gh-issue-142332.PNvXCV.rst diff --git a/Lib/argparse.py b/Lib/argparse.py index 07d7d77e8845be..27a63728eb4064 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -467,16 +467,12 @@ def _get_actions_usage_parts_with_split(self, actions, groups, opt_count=None): # produce all arg strings elif not action.option_strings: default = self._get_default_metavar_for_positional(action) - part = ( - t.summary_action - + self._format_args(action, default) - + t.reset - ) - + part = self._format_args(action, default) # if it's in a group, strip the outer [] if action in group_actions: if part[0] == '[' and part[-1] == ']': part = part[1:-1] + part = t.summary_action + part + t.reset # produce the first way to invoke the option in brackets else: diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index dff7ba750fa559..041d3671706193 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -7361,7 +7361,28 @@ def test_argparse_color(self): ), ) - def test_argparse_color_usage(self): + def test_argparse_color_mutually_exclusive_group_usage(self): + parser = argparse.ArgumentParser(color=True, prog="PROG") + group = parser.add_mutually_exclusive_group() + group.add_argument('--foo', action='store_true', help='FOO') + group.add_argument('--spam', help='SPAM') + group.add_argument('badger', nargs='*', help='BADGER') + + prog = self.theme.prog + heading = self.theme.heading + long = self.theme.summary_long_option + short = self.theme.summary_short_option + label = self.theme.summary_label + pos = self.theme.summary_action + reset = self.theme.reset + + self.assertEqual(parser.format_usage(), + f"{heading}usage: {reset}{prog}PROG{reset} [{short}-h{reset}] " + f"[{long}--foo{reset} | " + f"{long}--spam {label}SPAM{reset} | " + f"{pos}badger ...{reset}]\n") + + def test_argparse_color_custom_usage(self): # Arrange parser = argparse.ArgumentParser( add_help=False, diff --git a/Misc/NEWS.d/next/Library/2025-12-06-13-02-13.gh-issue-142332.PNvXCV.rst b/Misc/NEWS.d/next/Library/2025-12-06-13-02-13.gh-issue-142332.PNvXCV.rst new file mode 100644 index 00000000000000..ee2d5e1d4911a7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-06-13-02-13.gh-issue-142332.PNvXCV.rst @@ -0,0 +1,2 @@ +Fix usage formatting for positional arguments in mutually exclusive groups in :mod:`argparse`. +in :mod:`argparse`. From 0ed56ed88fc0ed3954cf601d116c7ba134910567 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Sat, 6 Dec 2025 10:30:50 -0800 Subject: [PATCH 277/638] GH-64532: Include parent's required optional arguments in subparser usage (#142355) --- Lib/argparse.py | 6 ++++-- Lib/test/test_argparse.py | 10 ++++++++++ .../2025-12-06-16-45-34.gh-issue-64532.4OXZpF.rst | 1 + 3 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-12-06-16-45-34.gh-issue-64532.4OXZpF.rst diff --git a/Lib/argparse.py b/Lib/argparse.py index 27a63728eb4064..9e2e076936cb51 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -2004,14 +2004,16 @@ def add_subparsers(self, **kwargs): self._subparsers = self._positionals # prog defaults to the usage message of this parser, skipping - # optional arguments and with no "usage:" prefix + # non-required optional arguments and with no "usage:" prefix if kwargs.get('prog') is None: # Create formatter without color to avoid storing ANSI codes in prog formatter = self.formatter_class(prog=self.prog) formatter._set_color(False) positionals = self._get_positional_actions() + required_optionals = [action for action in self._get_optional_actions() + if action.required] groups = self._mutually_exclusive_groups - formatter.add_usage(None, positionals, groups, '') + formatter.add_usage(None, required_optionals + positionals, groups, '') kwargs['prog'] = formatter.format_help().strip() # create the parsers action and add it to the positionals list diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 041d3671706193..248a92db74eb69 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -2770,6 +2770,16 @@ def test_optional_subparsers(self): ret = parser.parse_args(()) self.assertIsNone(ret.command) + def test_subparser_help_with_parent_required_optional(self): + parser = ErrorRaisingArgumentParser(prog='PROG') + parser.add_argument('--foo', required=True) + parser.add_argument('--bar') + subparsers = parser.add_subparsers() + parser_sub = subparsers.add_parser('sub') + parser_sub.add_argument('arg') + self.assertEqual(parser_sub.format_usage(), + 'usage: PROG --foo FOO sub [-h] arg\n') + def test_help(self): self.assertEqual(self.parser.format_usage(), 'usage: PROG [-h] [--foo] bar {1,2,3} ...\n') diff --git a/Misc/NEWS.d/next/Library/2025-12-06-16-45-34.gh-issue-64532.4OXZpF.rst b/Misc/NEWS.d/next/Library/2025-12-06-16-45-34.gh-issue-64532.4OXZpF.rst new file mode 100644 index 00000000000000..3bd950050aedf4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-06-16-45-34.gh-issue-64532.4OXZpF.rst @@ -0,0 +1 @@ +Subparser help now includes required optional arguments from the parent parser in the usage, making it clearer what arguments are needed to run a subcommand. Patch by Savannah Ostrowski. From 35142b18ae3ea0fa7bce04e69a938049ca3da70d Mon Sep 17 00:00:00 2001 From: Kir Chou <148194051+gkirchou@users.noreply.github.com> Date: Sun, 7 Dec 2025 03:59:52 +0900 Subject: [PATCH 278/638] gh-142168: explicitly initialize `stack_array` in `_PyEval_Vector` and `_PyEvalFramePushAndInit_Ex` (#142192) Co-authored-by: Kir Chou --- Python/ceval.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index 46bf644106ac39..aadc6369cbe520 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2344,7 +2344,7 @@ _PyEvalFramePushAndInit_Ex(PyThreadState *tstate, _PyStackRef func, PyObject *kwnames = NULL; _PyStackRef *newargs; PyObject *const *object_array = NULL; - _PyStackRef stack_array[8]; + _PyStackRef stack_array[8] = {0}; if (has_dict) { object_array = _PyStack_UnpackDict(tstate, _PyTuple_ITEMS(callargs), nargs, kwargs, &kwnames); if (object_array == NULL) { @@ -2407,7 +2407,7 @@ _PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func, if (kwnames) { total_args += PyTuple_GET_SIZE(kwnames); } - _PyStackRef stack_array[8]; + _PyStackRef stack_array[8] = {0}; _PyStackRef *arguments; if (total_args <= 8) { arguments = stack_array; From 56a442d0d893e41382dc740c50edc96df20d62f0 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Sat, 6 Dec 2025 11:31:40 -0800 Subject: [PATCH 279/638] GH-141565: Add async code awareness to Tachyon (#141533) Co-authored-by: Pablo Galindo Salgado --- Lib/profiling/sampling/cli.py | 41 +- Lib/profiling/sampling/collector.py | 97 ++- .../sampling/live_collector/collector.py | 116 +-- Lib/profiling/sampling/pstats_collector.py | 10 +- Lib/profiling/sampling/sample.py | 15 +- Lib/profiling/sampling/stack_collector.py | 16 +- .../test_sampling_profiler/mocks.py | 35 + .../test_sampling_profiler/test_async.py | 799 ++++++++++++++++++ .../test_sampling_profiler/test_cli.py | 162 ++++ .../test_integration.py | 125 +++ ...-11-14-18-00-41.gh-issue-141565.Ap2bhJ.rst | 1 + Modules/_remote_debugging/_remote_debugging.h | 1 + Modules/_remote_debugging/asyncio.c | 22 + Modules/_remote_debugging/module.c | 8 +- 14 files changed, 1360 insertions(+), 88 deletions(-) create mode 100644 Lib/test/test_profiling/test_sampling_profiler/test_async.py create mode 100644 Misc/NEWS.d/next/Library/2025-11-14-18-00-41.gh-issue-141565.Ap2bhJ.rst diff --git a/Lib/profiling/sampling/cli.py b/Lib/profiling/sampling/cli.py index 5c0e39d77371ef..0a082c0c6386ee 100644 --- a/Lib/profiling/sampling/cli.py +++ b/Lib/profiling/sampling/cli.py @@ -195,6 +195,11 @@ def _add_sampling_options(parser): dest="gc", help='Don\'t include artificial "" frames to denote active garbage collection', ) + sampling_group.add_argument( + "--async-aware", + action="store_true", + help="Enable async-aware profiling (uses task-based stack reconstruction)", + ) def _add_mode_options(parser): @@ -205,7 +210,14 @@ def _add_mode_options(parser): choices=["wall", "cpu", "gil"], default="wall", help="Sampling mode: wall (all samples), cpu (only samples when thread is on CPU), " - "gil (only samples when thread holds the GIL)", + "gil (only samples when thread holds the GIL). Incompatible with --async-aware", + ) + mode_group.add_argument( + "--async-mode", + choices=["running", "all"], + default="running", + help='Async profiling mode: "running" (only running task) ' + 'or "all" (all tasks including waiting). Requires --async-aware', ) @@ -382,6 +394,27 @@ def _validate_args(args, parser): "Live mode requires the curses module, which is not available." ) + # Async-aware mode is incompatible with --native, --no-gc, --mode, and --all-threads + if args.async_aware: + issues = [] + if args.native: + issues.append("--native") + if not args.gc: + issues.append("--no-gc") + if hasattr(args, 'mode') and args.mode != "wall": + issues.append(f"--mode={args.mode}") + if hasattr(args, 'all_threads') and args.all_threads: + issues.append("--all-threads") + if issues: + parser.error( + f"Options {', '.join(issues)} are incompatible with --async-aware. " + "Async-aware profiling uses task-based stack reconstruction." + ) + + # --async-mode requires --async-aware + if hasattr(args, 'async_mode') and args.async_mode != "running" and not args.async_aware: + parser.error("--async-mode requires --async-aware to be enabled.") + # Live mode is incompatible with format options if hasattr(args, 'live') and args.live: if args.format != "pstats": @@ -570,6 +603,7 @@ def _handle_attach(args): all_threads=args.all_threads, realtime_stats=args.realtime_stats, mode=mode, + async_aware=args.async_mode if args.async_aware else None, native=args.native, gc=args.gc, ) @@ -618,6 +652,7 @@ def _handle_run(args): all_threads=args.all_threads, realtime_stats=args.realtime_stats, mode=mode, + async_aware=args.async_mode if args.async_aware else None, native=args.native, gc=args.gc, ) @@ -650,6 +685,7 @@ def _handle_live_attach(args, pid): limit=20, # Default limit pid=pid, mode=mode, + async_aware=args.async_mode if args.async_aware else None, ) # Sample in live mode @@ -660,6 +696,7 @@ def _handle_live_attach(args, pid): all_threads=args.all_threads, realtime_stats=args.realtime_stats, mode=mode, + async_aware=args.async_mode if args.async_aware else None, native=args.native, gc=args.gc, ) @@ -689,6 +726,7 @@ def _handle_live_run(args): limit=20, # Default limit pid=process.pid, mode=mode, + async_aware=args.async_mode if args.async_aware else None, ) # Profile the subprocess in live mode @@ -700,6 +738,7 @@ def _handle_live_run(args): all_threads=args.all_threads, realtime_stats=args.realtime_stats, mode=mode, + async_aware=args.async_mode if args.async_aware else None, native=args.native, gc=args.gc, ) diff --git a/Lib/profiling/sampling/collector.py b/Lib/profiling/sampling/collector.py index 6187f351cb596b..f63ea0afd8ac0a 100644 --- a/Lib/profiling/sampling/collector.py +++ b/Lib/profiling/sampling/collector.py @@ -2,10 +2,16 @@ from .constants import ( THREAD_STATUS_HAS_GIL, THREAD_STATUS_ON_CPU, - THREAD_STATUS_UNKNOWN, THREAD_STATUS_GIL_REQUESTED, + THREAD_STATUS_UNKNOWN, ) +try: + from _remote_debugging import FrameInfo +except ImportError: + # Fallback definition if _remote_debugging is not available + FrameInfo = None + class Collector(ABC): @abstractmethod def collect(self, stack_frames): @@ -33,6 +39,95 @@ def _iter_all_frames(self, stack_frames, skip_idle=False): if frames: yield frames, thread_info.thread_id + def _iter_async_frames(self, awaited_info_list): + # Phase 1: Index tasks and build parent relationships with pre-computed selection + task_map, child_to_parent, all_task_ids, all_parent_ids = self._build_task_graph(awaited_info_list) + + # Phase 2: Find leaf tasks (tasks not awaited by anyone) + leaf_task_ids = self._find_leaf_tasks(all_task_ids, all_parent_ids) + + # Phase 3: Build linear stacks from each leaf to root (optimized - no sorting!) + yield from self._build_linear_stacks(leaf_task_ids, task_map, child_to_parent) + + def _build_task_graph(self, awaited_info_list): + task_map = {} + child_to_parent = {} # Maps child_id -> (selected_parent_id, parent_count) + all_task_ids = set() + all_parent_ids = set() # Track ALL parent IDs for leaf detection + + for awaited_info in awaited_info_list: + thread_id = awaited_info.thread_id + for task_info in awaited_info.awaited_by: + task_id = task_info.task_id + task_map[task_id] = (task_info, thread_id) + all_task_ids.add(task_id) + + # Pre-compute selected parent and count for optimization + if task_info.awaited_by: + parent_ids = [p.task_name for p in task_info.awaited_by] + parent_count = len(parent_ids) + # Track ALL parents for leaf detection + all_parent_ids.update(parent_ids) + # Use min() for O(n) instead of sorted()[0] which is O(n log n) + selected_parent = min(parent_ids) if parent_count > 1 else parent_ids[0] + child_to_parent[task_id] = (selected_parent, parent_count) + + return task_map, child_to_parent, all_task_ids, all_parent_ids + + def _find_leaf_tasks(self, all_task_ids, all_parent_ids): + # Leaves are tasks that are not parents of any other task + return all_task_ids - all_parent_ids + + def _build_linear_stacks(self, leaf_task_ids, task_map, child_to_parent): + for leaf_id in leaf_task_ids: + frames = [] + visited = set() + current_id = leaf_id + thread_id = None + + # Follow the single parent chain from leaf to root + while current_id is not None: + # Cycle detection + if current_id in visited: + break + visited.add(current_id) + + # Check if task exists in task_map + if current_id not in task_map: + break + + task_info, tid = task_map[current_id] + + # Set thread_id from first task + if thread_id is None: + thread_id = tid + + # Add all frames from all coroutines in this task + if task_info.coroutine_stack: + for coro_info in task_info.coroutine_stack: + for frame in coro_info.call_stack: + frames.append(frame) + + # Get pre-computed parent info (no sorting needed!) + parent_info = child_to_parent.get(current_id) + + # Add task boundary marker with parent count annotation if multiple parents + task_name = task_info.task_name or "Task-" + str(task_info.task_id) + if parent_info: + selected_parent, parent_count = parent_info + if parent_count > 1: + task_name = f"{task_name} ({parent_count} parents)" + frames.append(FrameInfo(("", 0, task_name))) + current_id = selected_parent + else: + # Root task - no parent + frames.append(FrameInfo(("", 0, task_name))) + current_id = None + + # Yield the complete stack if we collected any frames + if frames and thread_id is not None: + yield frames, thread_id, leaf_id + def _is_gc_frame(self, frame): if isinstance(frame, tuple): funcname = frame[2] if len(frame) >= 3 else "" diff --git a/Lib/profiling/sampling/live_collector/collector.py b/Lib/profiling/sampling/live_collector/collector.py index 7adbf1bbe7f625..5edb02e6e88704 100644 --- a/Lib/profiling/sampling/live_collector/collector.py +++ b/Lib/profiling/sampling/live_collector/collector.py @@ -103,6 +103,7 @@ def __init__( pid=None, display=None, mode=None, + async_aware=None, ): """ Initialize the live stats collector. @@ -115,6 +116,7 @@ def __init__( pid: Process ID being profiled display: DisplayInterface implementation (None means curses will be used) mode: Profiling mode ('cpu', 'gil', etc.) - affects what stats are shown + async_aware: Async tracing mode - None (sync only), "all" or "running" """ self.result = collections.defaultdict( lambda: dict(total_rec_calls=0, direct_calls=0, cumulative_calls=0) @@ -133,6 +135,9 @@ def __init__( self.running = True self.pid = pid self.mode = mode # Profiling mode + self.async_aware = async_aware # Async tracing mode + # Pre-select frame iterator method to avoid per-call dispatch overhead + self._get_frame_iterator = self._get_async_frame_iterator if async_aware else self._get_sync_frame_iterator self._saved_stdout = None self._saved_stderr = None self._devnull = None @@ -294,6 +299,15 @@ def process_frames(self, frames, thread_id=None): if thread_data: thread_data.result[top_location]["direct_calls"] += 1 + def _get_sync_frame_iterator(self, stack_frames): + """Iterator for sync frames.""" + return self._iter_all_frames(stack_frames, skip_idle=self.skip_idle) + + def _get_async_frame_iterator(self, stack_frames): + """Iterator for async frames, yielding (frames, thread_id) tuples.""" + for frames, thread_id, task_id in self._iter_async_frames(stack_frames): + yield frames, thread_id + def collect_failed_sample(self): self.failed_samples += 1 self.total_samples += 1 @@ -304,78 +318,40 @@ def collect(self, stack_frames): self.start_time = time.perf_counter() self._last_display_update = self.start_time - # Thread status counts for this sample - temp_status_counts = { - "has_gil": 0, - "on_cpu": 0, - "gil_requested": 0, - "unknown": 0, - "total": 0, - } has_gc_frame = False - # Always collect data, even when paused - # Track thread status flags and GC frames - for interpreter_info in stack_frames: - threads = getattr(interpreter_info, "threads", []) - for thread_info in threads: - temp_status_counts["total"] += 1 - - # Track thread status using bit flags - status_flags = getattr(thread_info, "status", 0) - thread_id = getattr(thread_info, "thread_id", None) - - # Update aggregated counts - if status_flags & THREAD_STATUS_HAS_GIL: - temp_status_counts["has_gil"] += 1 - if status_flags & THREAD_STATUS_ON_CPU: - temp_status_counts["on_cpu"] += 1 - if status_flags & THREAD_STATUS_GIL_REQUESTED: - temp_status_counts["gil_requested"] += 1 - if status_flags & THREAD_STATUS_UNKNOWN: - temp_status_counts["unknown"] += 1 - - # Update per-thread status counts - if thread_id is not None: - thread_data = self._get_or_create_thread_data(thread_id) - thread_data.increment_status_flag(status_flags) - - # Process frames (respecting skip_idle) - if self.skip_idle: - has_gil = bool(status_flags & THREAD_STATUS_HAS_GIL) - on_cpu = bool(status_flags & THREAD_STATUS_ON_CPU) - if not (has_gil or on_cpu): - continue - - frames = getattr(thread_info, "frame_info", None) - if frames: - self.process_frames(frames, thread_id=thread_id) - - # Track thread IDs only for threads that actually have samples - if ( - thread_id is not None - and thread_id not in self.thread_ids - ): - self.thread_ids.append(thread_id) - - # Increment per-thread sample count and check for GC frames - thread_has_gc_frame = False - for frame in frames: - funcname = getattr(frame, "funcname", "") - if "" in funcname or "gc_collect" in funcname: - has_gc_frame = True - thread_has_gc_frame = True - break - - if thread_id is not None: - thread_data = self._get_or_create_thread_data(thread_id) - thread_data.sample_count += 1 - if thread_has_gc_frame: - thread_data.gc_frame_samples += 1 - - # Update cumulative thread status counts - for key, count in temp_status_counts.items(): - self.thread_status_counts[key] += count + # Collect thread status stats (only available in sync mode) + if not self.async_aware: + status_counts, sample_has_gc, per_thread_stats = self._collect_thread_status_stats(stack_frames) + for key, count in status_counts.items(): + self.thread_status_counts[key] += count + if sample_has_gc: + has_gc_frame = True + + for thread_id, stats in per_thread_stats.items(): + thread_data = self._get_or_create_thread_data(thread_id) + thread_data.has_gil += stats.get("has_gil", 0) + thread_data.on_cpu += stats.get("on_cpu", 0) + thread_data.gil_requested += stats.get("gil_requested", 0) + thread_data.unknown += stats.get("unknown", 0) + thread_data.total += stats.get("total", 0) + if stats.get("gc_samples", 0): + thread_data.gc_frame_samples += stats["gc_samples"] + + # Process frames using pre-selected iterator + for frames, thread_id in self._get_frame_iterator(stack_frames): + if not frames: + continue + + self.process_frames(frames, thread_id=thread_id) + + # Track thread IDs + if thread_id is not None and thread_id not in self.thread_ids: + self.thread_ids.append(thread_id) + + if thread_id is not None: + thread_data = self._get_or_create_thread_data(thread_id) + thread_data.sample_count += 1 if has_gc_frame: self.gc_frame_samples += 1 diff --git a/Lib/profiling/sampling/pstats_collector.py b/Lib/profiling/sampling/pstats_collector.py index b8b37a10c43ad3..4fe3acfa9ff80e 100644 --- a/Lib/profiling/sampling/pstats_collector.py +++ b/Lib/profiling/sampling/pstats_collector.py @@ -42,8 +42,14 @@ def _process_frames(self, frames): self.callers[callee][caller] += 1 def collect(self, stack_frames): - for frames, thread_id in self._iter_all_frames(stack_frames, skip_idle=self.skip_idle): - self._process_frames(frames) + if stack_frames and hasattr(stack_frames[0], "awaited_by"): + # Async frame processing + for frames, thread_id, task_id in self._iter_async_frames(stack_frames): + self._process_frames(frames) + else: + # Regular frame processing + for frames, thread_id in self._iter_all_frames(stack_frames, skip_idle=self.skip_idle): + self._process_frames(frames) def export(self, filename): self.create_stats() diff --git a/Lib/profiling/sampling/sample.py b/Lib/profiling/sampling/sample.py index 46fc1a05afaa74..99cac71a4049a6 100644 --- a/Lib/profiling/sampling/sample.py +++ b/Lib/profiling/sampling/sample.py @@ -48,7 +48,7 @@ def __init__(self, pid, sample_interval_usec, all_threads, *, mode=PROFILING_MOD self.total_samples = 0 self.realtime_stats = False - def sample(self, collector, duration_sec=10): + def sample(self, collector, duration_sec=10, *, async_aware=False): sample_interval_sec = self.sample_interval_usec / 1_000_000 running_time = 0 num_samples = 0 @@ -68,7 +68,12 @@ def sample(self, collector, duration_sec=10): current_time = time.perf_counter() if next_time < current_time: try: - stack_frames = self.unwinder.get_stack_trace() + if async_aware == "all": + stack_frames = self.unwinder.get_all_awaited_by() + elif async_aware == "running": + stack_frames = self.unwinder.get_async_stack_trace() + else: + stack_frames = self.unwinder.get_stack_trace() collector.collect(stack_frames) except ProcessLookupError: duration_sec = current_time - start_time @@ -191,6 +196,7 @@ def sample( all_threads=False, realtime_stats=False, mode=PROFILING_MODE_WALL, + async_aware=None, native=False, gc=True, ): @@ -233,7 +239,7 @@ def sample( profiler.realtime_stats = realtime_stats # Run the sampling - profiler.sample(collector, duration_sec) + profiler.sample(collector, duration_sec, async_aware=async_aware) return collector @@ -246,6 +252,7 @@ def sample_live( all_threads=False, realtime_stats=False, mode=PROFILING_MODE_WALL, + async_aware=None, native=False, gc=True, ): @@ -290,7 +297,7 @@ def sample_live( def curses_wrapper_func(stdscr): collector.init_curses(stdscr) try: - profiler.sample(collector, duration_sec) + profiler.sample(collector, duration_sec, async_aware=async_aware) # Mark as finished and keep the TUI running until user presses 'q' collector.mark_finished() # Keep processing input until user quits diff --git a/Lib/profiling/sampling/stack_collector.py b/Lib/profiling/sampling/stack_collector.py index e26536093130d1..1f766682858d45 100644 --- a/Lib/profiling/sampling/stack_collector.py +++ b/Lib/profiling/sampling/stack_collector.py @@ -17,10 +17,18 @@ def __init__(self, sample_interval_usec, *, skip_idle=False): self.skip_idle = skip_idle def collect(self, stack_frames, skip_idle=False): - for frames, thread_id in self._iter_all_frames(stack_frames, skip_idle=skip_idle): - if not frames: - continue - self.process_frames(frames, thread_id) + if stack_frames and hasattr(stack_frames[0], "awaited_by"): + # Async-aware mode: process async task frames + for frames, thread_id, task_id in self._iter_async_frames(stack_frames): + if not frames: + continue + self.process_frames(frames, thread_id) + else: + # Sync-only mode + for frames, thread_id in self._iter_all_frames(stack_frames, skip_idle=skip_idle): + if not frames: + continue + self.process_frames(frames, thread_id) def process_frames(self, frames, thread_id): pass diff --git a/Lib/test/test_profiling/test_sampling_profiler/mocks.py b/Lib/test/test_profiling/test_sampling_profiler/mocks.py index 9f1cd5b83e0856..7083362c7714f1 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/mocks.py +++ b/Lib/test/test_profiling/test_sampling_profiler/mocks.py @@ -36,3 +36,38 @@ def __init__(self, interpreter_id, threads): def __repr__(self): return f"MockInterpreterInfo(interpreter_id={self.interpreter_id}, threads={self.threads})" + + +class MockCoroInfo: + """Mock CoroInfo for testing async tasks.""" + + def __init__(self, task_name, call_stack): + self.task_name = task_name # In reality, this is the parent task ID + self.call_stack = call_stack + + def __repr__(self): + return f"MockCoroInfo(task_name={self.task_name}, call_stack={self.call_stack})" + + +class MockTaskInfo: + """Mock TaskInfo for testing async tasks.""" + + def __init__(self, task_id, task_name, coroutine_stack, awaited_by=None): + self.task_id = task_id + self.task_name = task_name + self.coroutine_stack = coroutine_stack # List of CoroInfo objects + self.awaited_by = awaited_by or [] # List of CoroInfo objects (parents) + + def __repr__(self): + return f"MockTaskInfo(task_id={self.task_id}, task_name={self.task_name})" + + +class MockAwaitedInfo: + """Mock AwaitedInfo for testing async tasks.""" + + def __init__(self, thread_id, awaited_by): + self.thread_id = thread_id + self.awaited_by = awaited_by # List of TaskInfo objects + + def __repr__(self): + return f"MockAwaitedInfo(thread_id={self.thread_id}, awaited_by={len(self.awaited_by)} tasks)" diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_async.py b/Lib/test/test_profiling/test_sampling_profiler/test_async.py new file mode 100644 index 00000000000000..d8ca86c996bffa --- /dev/null +++ b/Lib/test/test_profiling/test_sampling_profiler/test_async.py @@ -0,0 +1,799 @@ +"""Tests for async stack reconstruction in the sampling profiler. + +Each test covers a distinct algorithm path or edge case: +1. Graph building: _build_task_graph() +2. Leaf identification: _find_leaf_tasks() +3. Stack traversal: _build_linear_stacks() with BFS +""" + +import unittest + +try: + import _remote_debugging # noqa: F401 + from profiling.sampling.pstats_collector import PstatsCollector +except ImportError: + raise unittest.SkipTest( + "Test only runs when _remote_debugging is available" + ) + +from .mocks import MockFrameInfo, MockCoroInfo, MockTaskInfo, MockAwaitedInfo + + +class TestAsyncStackReconstruction(unittest.TestCase): + """Test async task tree linear stack reconstruction algorithm.""" + + def test_empty_input(self): + """Test _build_task_graph with empty awaited_info_list.""" + collector = PstatsCollector(sample_interval_usec=1000) + stacks = list(collector._iter_async_frames([])) + self.assertEqual(len(stacks), 0) + + def test_single_root_task(self): + """Test _find_leaf_tasks: root task with no parents is its own leaf.""" + collector = PstatsCollector(sample_interval_usec=1000) + + root = MockTaskInfo( + task_id=123, + task_name="Task-1", + coroutine_stack=[ + MockCoroInfo( + task_name="Task-1", + call_stack=[MockFrameInfo("main.py", 10, "main")] + ) + ], + awaited_by=[] + ) + + awaited_info_list = [MockAwaitedInfo(thread_id=100, awaited_by=[root])] + stacks = list(collector._iter_async_frames(awaited_info_list)) + + # Single root is both leaf and root + self.assertEqual(len(stacks), 1) + frames, thread_id, leaf_id = stacks[0] + self.assertEqual(leaf_id, 123) + self.assertEqual(thread_id, 100) + + def test_parent_child_chain(self): + """Test _build_linear_stacks: BFS follows parent links from leaf to root. + + Task graph: + + Parent (id=1) + | + Child (id=2) + """ + collector = PstatsCollector(sample_interval_usec=1000) + + child = MockTaskInfo( + task_id=2, + task_name="Child", + coroutine_stack=[ + MockCoroInfo(task_name="Child", call_stack=[MockFrameInfo("c.py", 5, "child_fn")]) + ], + awaited_by=[ + MockCoroInfo(task_name=1, call_stack=[MockFrameInfo("p.py", 10, "parent_await")]) + ] + ) + + parent = MockTaskInfo( + task_id=1, + task_name="Parent", + coroutine_stack=[ + MockCoroInfo(task_name="Parent", call_stack=[MockFrameInfo("p.py", 15, "parent_fn")]) + ], + awaited_by=[] + ) + + awaited_info_list = [MockAwaitedInfo(thread_id=200, awaited_by=[child, parent])] + stacks = list(collector._iter_async_frames(awaited_info_list)) + + # Leaf is child, traverses to parent + self.assertEqual(len(stacks), 1) + frames, thread_id, leaf_id = stacks[0] + self.assertEqual(leaf_id, 2) + + # Verify both child and parent frames present + func_names = [f.funcname for f in frames] + self.assertIn("child_fn", func_names) + self.assertIn("parent_fn", func_names) + + def test_multiple_leaf_tasks(self): + """Test _find_leaf_tasks: identifies multiple leaves correctly. + + Task graph (fan-out from root): + + Root (id=1) + / \ + Leaf1 (id=10) Leaf2 (id=20) + + Expected: 2 stacks (one for each leaf). + """ + collector = PstatsCollector(sample_interval_usec=1000) + leaf1 = MockTaskInfo( + task_id=10, + task_name="Leaf1", + coroutine_stack=[MockCoroInfo(task_name="Leaf1", call_stack=[MockFrameInfo("l1.py", 1, "f1")])], + awaited_by=[MockCoroInfo(task_name=1, call_stack=[MockFrameInfo("r.py", 5, "root")])] + ) + + leaf2 = MockTaskInfo( + task_id=20, + task_name="Leaf2", + coroutine_stack=[MockCoroInfo(task_name="Leaf2", call_stack=[MockFrameInfo("l2.py", 2, "f2")])], + awaited_by=[MockCoroInfo(task_name=1, call_stack=[MockFrameInfo("r.py", 5, "root")])] + ) + + root = MockTaskInfo( + task_id=1, + task_name="Root", + coroutine_stack=[MockCoroInfo(task_name="Root", call_stack=[MockFrameInfo("r.py", 10, "main")])], + awaited_by=[] + ) + + awaited_info_list = [MockAwaitedInfo(thread_id=300, awaited_by=[leaf1, leaf2, root])] + stacks = list(collector._iter_async_frames(awaited_info_list)) + + # Two leaves = two stacks + self.assertEqual(len(stacks), 2) + leaf_ids = {leaf_id for _, _, leaf_id in stacks} + self.assertEqual(leaf_ids, {10, 20}) + + def test_cycle_detection(self): + """Test _build_linear_stacks: cycle detection prevents infinite loops. + + Task graph (cyclic dependency): + + A (id=1) <---> B (id=2) + + Neither task is a leaf (both have parents), so no stacks are produced. + """ + collector = PstatsCollector(sample_interval_usec=1000) + task_a = MockTaskInfo( + task_id=1, + task_name="A", + coroutine_stack=[MockCoroInfo(task_name="A", call_stack=[MockFrameInfo("a.py", 1, "a")])], + awaited_by=[MockCoroInfo(task_name=2, call_stack=[MockFrameInfo("b.py", 5, "b")])] + ) + + task_b = MockTaskInfo( + task_id=2, + task_name="B", + coroutine_stack=[MockCoroInfo(task_name="B", call_stack=[MockFrameInfo("b.py", 10, "b")])], + awaited_by=[MockCoroInfo(task_name=1, call_stack=[MockFrameInfo("a.py", 15, "a")])] + ) + + awaited_info_list = [MockAwaitedInfo(thread_id=400, awaited_by=[task_a, task_b])] + stacks = list(collector._iter_async_frames(awaited_info_list)) + + # No leaves (both have parents), should return empty + self.assertEqual(len(stacks), 0) + + def test_orphaned_parent_reference(self): + """Test _build_linear_stacks: handles parent ID not in task_map.""" + collector = PstatsCollector(sample_interval_usec=1000) + + # Task references non-existent parent + orphan = MockTaskInfo( + task_id=5, + task_name="Orphan", + coroutine_stack=[MockCoroInfo(task_name="Orphan", call_stack=[MockFrameInfo("o.py", 1, "orphan")])], + awaited_by=[MockCoroInfo(task_name=999, call_stack=[])] # 999 doesn't exist + ) + + awaited_info_list = [MockAwaitedInfo(thread_id=500, awaited_by=[orphan])] + stacks = list(collector._iter_async_frames(awaited_info_list)) + + # Stops at missing parent, yields what it has + self.assertEqual(len(stacks), 1) + frames, _, leaf_id = stacks[0] + self.assertEqual(leaf_id, 5) + + def test_multiple_coroutines_per_task(self): + """Test _build_linear_stacks: collects frames from all coroutines in task.""" + collector = PstatsCollector(sample_interval_usec=1000) + + # Task with multiple coroutines (e.g., nested async generators) + task = MockTaskInfo( + task_id=7, + task_name="Multi", + coroutine_stack=[ + MockCoroInfo(task_name="Multi", call_stack=[MockFrameInfo("g.py", 5, "gen1")]), + MockCoroInfo(task_name="Multi", call_stack=[MockFrameInfo("g.py", 10, "gen2")]), + ], + awaited_by=[] + ) + + awaited_info_list = [MockAwaitedInfo(thread_id=600, awaited_by=[task])] + stacks = list(collector._iter_async_frames(awaited_info_list)) + + self.assertEqual(len(stacks), 1) + frames, _, _ = stacks[0] + + # Both coroutine frames should be present + func_names = [f.funcname for f in frames] + self.assertIn("gen1", func_names) + self.assertIn("gen2", func_names) + + def test_multiple_threads(self): + """Test _build_task_graph: handles multiple AwaitedInfo (different threads).""" + collector = PstatsCollector(sample_interval_usec=1000) + + # Two threads with separate task trees + thread1_task = MockTaskInfo( + task_id=100, + task_name="T1", + coroutine_stack=[MockCoroInfo(task_name="T1", call_stack=[MockFrameInfo("t1.py", 1, "t1")])], + awaited_by=[] + ) + + thread2_task = MockTaskInfo( + task_id=200, + task_name="T2", + coroutine_stack=[MockCoroInfo(task_name="T2", call_stack=[MockFrameInfo("t2.py", 1, "t2")])], + awaited_by=[] + ) + + awaited_info_list = [ + MockAwaitedInfo(thread_id=1, awaited_by=[thread1_task]), + MockAwaitedInfo(thread_id=2, awaited_by=[thread2_task]), + ] + + stacks = list(collector._iter_async_frames(awaited_info_list)) + + # Two threads = two stacks + self.assertEqual(len(stacks), 2) + + # Verify thread IDs preserved + thread_ids = {thread_id for _, thread_id, _ in stacks} + self.assertEqual(thread_ids, {1, 2}) + + def test_collect_public_interface(self): + """Test collect() method correctly routes to async frame processing.""" + collector = PstatsCollector(sample_interval_usec=1000) + + child = MockTaskInfo( + task_id=50, + task_name="Child", + coroutine_stack=[MockCoroInfo(task_name="Child", call_stack=[MockFrameInfo("c.py", 1, "child")])], + awaited_by=[MockCoroInfo(task_name=51, call_stack=[])] + ) + + parent = MockTaskInfo( + task_id=51, + task_name="Parent", + coroutine_stack=[MockCoroInfo(task_name="Parent", call_stack=[MockFrameInfo("p.py", 1, "parent")])], + awaited_by=[] + ) + + awaited_info_list = [MockAwaitedInfo(thread_id=999, awaited_by=[child, parent])] + + # Public interface: collect() + collector.collect(awaited_info_list) + + # Verify stats collected + self.assertGreater(len(collector.result), 0) + func_names = [loc[2] for loc in collector.result.keys()] + self.assertIn("child", func_names) + self.assertIn("parent", func_names) + + def test_diamond_pattern_multiple_parents(self): + """Test _build_linear_stacks: task with 2+ parents picks one deterministically. + + CRITICAL: Tests that when a task has multiple parents, we pick one parent + deterministically (sorted, first one) and annotate the task name with parent count. + """ + collector = PstatsCollector(sample_interval_usec=1000) + + # Diamond pattern: Root spawns A and B, both await Child + # + # Root (id=1) + # / \ + # A (id=2) B (id=3) + # \ / + # Child (id=4) + # + + child = MockTaskInfo( + task_id=4, + task_name="Child", + coroutine_stack=[MockCoroInfo(task_name="Child", call_stack=[MockFrameInfo("c.py", 1, "child_work")])], + awaited_by=[ + MockCoroInfo(task_name=2, call_stack=[MockFrameInfo("a.py", 5, "a_await")]), # Parent A + MockCoroInfo(task_name=3, call_stack=[MockFrameInfo("b.py", 5, "b_await")]), # Parent B + ] + ) + + parent_a = MockTaskInfo( + task_id=2, + task_name="A", + coroutine_stack=[MockCoroInfo(task_name="A", call_stack=[MockFrameInfo("a.py", 10, "a_work")])], + awaited_by=[MockCoroInfo(task_name=1, call_stack=[MockFrameInfo("root.py", 5, "root_spawn")])] + ) + + parent_b = MockTaskInfo( + task_id=3, + task_name="B", + coroutine_stack=[MockCoroInfo(task_name="B", call_stack=[MockFrameInfo("b.py", 10, "b_work")])], + awaited_by=[MockCoroInfo(task_name=1, call_stack=[MockFrameInfo("root.py", 5, "root_spawn")])] + ) + + root = MockTaskInfo( + task_id=1, + task_name="Root", + coroutine_stack=[MockCoroInfo(task_name="Root", call_stack=[MockFrameInfo("root.py", 20, "main")])], + awaited_by=[] + ) + + awaited_info_list = [MockAwaitedInfo(thread_id=777, awaited_by=[child, parent_a, parent_b, root])] + stacks = list(collector._iter_async_frames(awaited_info_list)) + + # Should get 1 stack: Child->A->Root (picks parent with lowest ID: 2) + self.assertEqual(len(stacks), 1, "Diamond should create only 1 path, picking first sorted parent") + + # Verify the single stack + frames, thread_id, leaf_id = stacks[0] + self.assertEqual(leaf_id, 4) + self.assertEqual(thread_id, 777) + + func_names = [f.funcname for f in frames] + # Stack should contain child, parent A (id=2, first when sorted), and root + self.assertIn("child_work", func_names) + self.assertIn("a_work", func_names, "Should use parent A (id=2, first when sorted)") + self.assertNotIn("b_work", func_names, "Should not include parent B") + self.assertIn("main", func_names) + + # Verify Child task is annotated with parent count + self.assertIn("Child (2 parents)", func_names, "Child task should be annotated with parent count") + + def test_empty_coroutine_stack(self): + """Test _build_linear_stacks: handles empty coroutine_stack (line 109 condition false).""" + collector = PstatsCollector(sample_interval_usec=1000) + + # Task with no coroutine_stack + task = MockTaskInfo( + task_id=99, + task_name="EmptyStack", + coroutine_stack=[], # Empty! + awaited_by=[] + ) + + awaited_info_list = [MockAwaitedInfo(thread_id=111, awaited_by=[task])] + stacks = list(collector._iter_async_frames(awaited_info_list)) + + self.assertEqual(len(stacks), 1) + frames, _, _ = stacks[0] + + # Should only have task marker, no function frames + func_names = [f.funcname for f in frames] + self.assertEqual(len(func_names), 1, "Should only have task marker") + self.assertIn("EmptyStack", func_names) + + def test_orphaned_parent_with_no_frames_collected(self): + """Test _build_linear_stacks: orphaned parent at start with empty frames (line 94-96).""" + collector = PstatsCollector(sample_interval_usec=1000) + + # Leaf that doesn't exist in task_map (should not happen normally, but test robustness) + # We'll create a scenario where the leaf_id is present but empty + + # Task references non-existent parent, and has no coroutine_stack + orphan = MockTaskInfo( + task_id=88, + task_name="Orphan", + coroutine_stack=[], # No frames + awaited_by=[MockCoroInfo(task_name=999, call_stack=[])] # Parent doesn't exist + ) + + awaited_info_list = [MockAwaitedInfo(thread_id=222, awaited_by=[orphan])] + stacks = list(collector._iter_async_frames(awaited_info_list)) + + # Should yield because we have the task marker even with no function frames + self.assertEqual(len(stacks), 1) + frames, _, leaf_id = stacks[0] + self.assertEqual(leaf_id, 88) + # Has task marker but no function frames + self.assertGreater(len(frames), 0, "Should have at least task marker") + + def test_frame_ordering(self): + """Test _build_linear_stacks: frames are collected in correct order (leaf->root). + + Task graph (3-level chain): + + Root (id=1) <- root_bottom, root_top + | + Middle (id=2) <- mid_bottom, mid_top + | + Leaf (id=3) <- leaf_bottom, leaf_top + + Expected frame order: leaf_bottom, leaf_top, mid_bottom, mid_top, root_bottom, root_top + (stack is built bottom-up: leaf frames first, then parent frames). + """ + collector = PstatsCollector(sample_interval_usec=1000) + leaf = MockTaskInfo( + task_id=3, + task_name="Leaf", + coroutine_stack=[ + MockCoroInfo(task_name="Leaf", call_stack=[ + MockFrameInfo("leaf.py", 1, "leaf_bottom"), + MockFrameInfo("leaf.py", 2, "leaf_top"), + ]) + ], + awaited_by=[MockCoroInfo(task_name=2, call_stack=[])] + ) + + middle = MockTaskInfo( + task_id=2, + task_name="Middle", + coroutine_stack=[ + MockCoroInfo(task_name="Middle", call_stack=[ + MockFrameInfo("mid.py", 1, "mid_bottom"), + MockFrameInfo("mid.py", 2, "mid_top"), + ]) + ], + awaited_by=[MockCoroInfo(task_name=1, call_stack=[])] + ) + + root = MockTaskInfo( + task_id=1, + task_name="Root", + coroutine_stack=[ + MockCoroInfo(task_name="Root", call_stack=[ + MockFrameInfo("root.py", 1, "root_bottom"), + MockFrameInfo("root.py", 2, "root_top"), + ]) + ], + awaited_by=[] + ) + + awaited_info_list = [MockAwaitedInfo(thread_id=333, awaited_by=[leaf, middle, root])] + stacks = list(collector._iter_async_frames(awaited_info_list)) + + self.assertEqual(len(stacks), 1) + frames, _, _ = stacks[0] + + func_names = [f.funcname for f in frames] + + # Order should be: leaf frames, leaf marker, middle frames, middle marker, root frames, root marker + leaf_bottom_idx = func_names.index("leaf_bottom") + leaf_top_idx = func_names.index("leaf_top") + mid_bottom_idx = func_names.index("mid_bottom") + root_bottom_idx = func_names.index("root_bottom") + + # Verify leaf comes before middle comes before root + self.assertLess(leaf_bottom_idx, leaf_top_idx, "Leaf frames in order") + self.assertLess(leaf_top_idx, mid_bottom_idx, "Leaf before middle") + self.assertLess(mid_bottom_idx, root_bottom_idx, "Middle before root") + + def test_complex_multi_parent_convergence(self): + """Test _build_linear_stacks: multiple leaves with same parents pick deterministically. + + Tests that when multiple leaves have multiple parents, each leaf picks the same + parent (sorted, first one) and all leaves are annotated with parent count. + + Task graph structure (both leaves awaited by both A and B):: + + Root (id=1) + / \\ + A (id=2) B (id=3) + | \\ / | + | \\ / | + | \\/ | + | /\\ | + | / \\ | + LeafX (id=4) LeafY (id=5) + + Expected behavior: Both leaves pick parent A (lowest id=2) for their stack path. + Result: 2 stacks, both going through A -> Root (B is skipped). + """ + collector = PstatsCollector(sample_interval_usec=1000) + + leaf_x = MockTaskInfo( + task_id=4, + task_name="LeafX", + coroutine_stack=[MockCoroInfo(task_name="LeafX", call_stack=[MockFrameInfo("x.py", 1, "x")])], + awaited_by=[ + MockCoroInfo(task_name=2, call_stack=[]), + MockCoroInfo(task_name=3, call_stack=[]), + ] + ) + + leaf_y = MockTaskInfo( + task_id=5, + task_name="LeafY", + coroutine_stack=[MockCoroInfo(task_name="LeafY", call_stack=[MockFrameInfo("y.py", 1, "y")])], + awaited_by=[ + MockCoroInfo(task_name=2, call_stack=[]), + MockCoroInfo(task_name=3, call_stack=[]), + ] + ) + + parent_a = MockTaskInfo( + task_id=2, + task_name="A", + coroutine_stack=[MockCoroInfo(task_name="A", call_stack=[MockFrameInfo("a.py", 1, "a")])], + awaited_by=[MockCoroInfo(task_name=1, call_stack=[])] + ) + + parent_b = MockTaskInfo( + task_id=3, + task_name="B", + coroutine_stack=[MockCoroInfo(task_name="B", call_stack=[MockFrameInfo("b.py", 1, "b")])], + awaited_by=[MockCoroInfo(task_name=1, call_stack=[])] + ) + + root = MockTaskInfo( + task_id=1, + task_name="Root", + coroutine_stack=[MockCoroInfo(task_name="Root", call_stack=[MockFrameInfo("r.py", 1, "root")])], + awaited_by=[] + ) + + awaited_info_list = [MockAwaitedInfo(thread_id=444, awaited_by=[leaf_x, leaf_y, parent_a, parent_b, root])] + stacks = list(collector._iter_async_frames(awaited_info_list)) + + # 2 leaves, each picks same parent (A, id=2) = 2 paths + self.assertEqual(len(stacks), 2, "Should create 2 paths: X->A->Root, Y->A->Root") + + # Verify both leaves pick parent A (id=2, first when sorted) + leaf_ids_seen = set() + for frames, _, leaf_id in stacks: + leaf_ids_seen.add(leaf_id) + func_names = [f.funcname for f in frames] + + # Both stacks should go through parent A only + self.assertIn("a", func_names, "Should use parent A (id=2, first when sorted)") + self.assertNotIn("b", func_names, "Should not include parent B") + self.assertIn("root", func_names, "Should reach root") + + # Check for parent count annotation on the leaf + if leaf_id == 4: + self.assertIn("x", func_names) + self.assertIn("LeafX (2 parents)", func_names, "LeafX should be annotated with parent count") + elif leaf_id == 5: + self.assertIn("y", func_names) + self.assertIn("LeafY (2 parents)", func_names, "LeafY should be annotated with parent count") + + # Both leaves should be represented + self.assertEqual(leaf_ids_seen, {4, 5}, "Both LeafX and LeafY should have paths") + + +class TestFlamegraphCollectorAsync(unittest.TestCase): + """Test FlamegraphCollector with async frames.""" + + def test_flamegraph_with_async_frames(self): + """Test FlamegraphCollector correctly processes async task frames.""" + from profiling.sampling.stack_collector import FlamegraphCollector + + collector = FlamegraphCollector(sample_interval_usec=1000) + + # Build async task tree: Root -> Child + child = MockTaskInfo( + task_id=2, + task_name="ChildTask", + coroutine_stack=[ + MockCoroInfo( + task_name="ChildTask", + call_stack=[MockFrameInfo("child.py", 10, "child_work")] + ) + ], + awaited_by=[MockCoroInfo(task_name=1, call_stack=[])] + ) + + root = MockTaskInfo( + task_id=1, + task_name="RootTask", + coroutine_stack=[ + MockCoroInfo( + task_name="RootTask", + call_stack=[MockFrameInfo("root.py", 20, "root_work")] + ) + ], + awaited_by=[] + ) + + awaited_info_list = [MockAwaitedInfo(thread_id=100, awaited_by=[child, root])] + + # Collect async frames + collector.collect(awaited_info_list) + + # Verify samples were collected + self.assertGreater(collector._total_samples, 0) + + # Verify the flamegraph tree structure contains our functions + root_node = collector._root + self.assertGreater(root_node["samples"], 0) + + # Check that thread ID was tracked + self.assertIn(100, collector._all_threads) + + def test_flamegraph_with_task_markers(self): + """Test FlamegraphCollector includes boundary markers.""" + from profiling.sampling.stack_collector import FlamegraphCollector + + collector = FlamegraphCollector(sample_interval_usec=1000) + + task = MockTaskInfo( + task_id=42, + task_name="MyTask", + coroutine_stack=[ + MockCoroInfo( + task_name="MyTask", + call_stack=[MockFrameInfo("work.py", 5, "do_work")] + ) + ], + awaited_by=[] + ) + + awaited_info_list = [MockAwaitedInfo(thread_id=200, awaited_by=[task])] + collector.collect(awaited_info_list) + + # Find marker in the tree + def find_task_marker(node, depth=0): + for func, child in node.get("children", {}).items(): + if func[0] == "": + return func + result = find_task_marker(child, depth + 1) + if result: + return result + return None + + task_marker = find_task_marker(collector._root) + self.assertIsNotNone(task_marker, "Should have marker in tree") + self.assertEqual(task_marker[0], "") + self.assertIn("MyTask", task_marker[2]) + + def test_flamegraph_multiple_async_samples(self): + """Test FlamegraphCollector aggregates multiple async samples correctly.""" + from profiling.sampling.stack_collector import FlamegraphCollector + + collector = FlamegraphCollector(sample_interval_usec=1000) + + task = MockTaskInfo( + task_id=1, + task_name="Task", + coroutine_stack=[ + MockCoroInfo( + task_name="Task", + call_stack=[MockFrameInfo("work.py", 10, "work")] + ) + ], + awaited_by=[] + ) + + awaited_info_list = [MockAwaitedInfo(thread_id=300, awaited_by=[task])] + + # Collect multiple samples + for _ in range(5): + collector.collect(awaited_info_list) + + # Verify sample count + self.assertEqual(collector._sample_count, 5) + self.assertEqual(collector._total_samples, 5) + + +class TestAsyncAwareParameterFlow(unittest.TestCase): + """Integration tests for async_aware parameter flow from CLI to unwinder.""" + + def test_sample_function_accepts_async_aware(self): + """Test that sample() function accepts async_aware parameter.""" + from profiling.sampling.sample import sample + import inspect + + sig = inspect.signature(sample) + self.assertIn("async_aware", sig.parameters) + + def test_sample_live_function_accepts_async_aware(self): + """Test that sample_live() function accepts async_aware parameter.""" + from profiling.sampling.sample import sample_live + import inspect + + sig = inspect.signature(sample_live) + self.assertIn("async_aware", sig.parameters) + + def test_sample_profiler_sample_accepts_async_aware(self): + """Test that SampleProfiler.sample() accepts async_aware parameter.""" + from profiling.sampling.sample import SampleProfiler + import inspect + + sig = inspect.signature(SampleProfiler.sample) + self.assertIn("async_aware", sig.parameters) + + def test_async_aware_all_sees_sleeping_and_running_tasks(self): + """Test async_aware='all' captures both sleeping and CPU-running tasks.""" + # Sleeping task (awaiting) + sleeping_task = MockTaskInfo( + task_id=1, + task_name="SleepingTask", + coroutine_stack=[ + MockCoroInfo( + task_name="SleepingTask", + call_stack=[MockFrameInfo("sleeper.py", 10, "sleep_work")] + ) + ], + awaited_by=[] + ) + + # CPU-running task (active) + running_task = MockTaskInfo( + task_id=2, + task_name="RunningTask", + coroutine_stack=[ + MockCoroInfo( + task_name="RunningTask", + call_stack=[MockFrameInfo("runner.py", 20, "cpu_work")] + ) + ], + awaited_by=[] + ) + + # Both tasks returned by get_all_awaited_by + awaited_info_list = [MockAwaitedInfo(thread_id=100, awaited_by=[sleeping_task, running_task])] + + collector = PstatsCollector(sample_interval_usec=1000) + collector.collect(awaited_info_list) + collector.create_stats() + + # Both tasks should be visible + sleeping_key = ("sleeper.py", 10, "sleep_work") + running_key = ("runner.py", 20, "cpu_work") + + self.assertIn(sleeping_key, collector.stats) + self.assertIn(running_key, collector.stats) + + # Task markers should also be present + task_keys = [k for k in collector.stats if k[0] == ""] + self.assertGreater(len(task_keys), 0, "Should have markers in stats") + + # Verify task names are in the markers + task_names = [k[2] for k in task_keys] + self.assertTrue( + any("SleepingTask" in name for name in task_names), + "SleepingTask should be in task markers" + ) + self.assertTrue( + any("RunningTask" in name for name in task_names), + "RunningTask should be in task markers" + ) + + def test_async_aware_running_sees_only_running_task(self): + """Test async_aware='running' only shows the currently running task stack.""" + # Only the running task's stack is returned by get_async_stack_trace + running_task = MockTaskInfo( + task_id=2, + task_name="RunningTask", + coroutine_stack=[ + MockCoroInfo( + task_name="RunningTask", + call_stack=[MockFrameInfo("runner.py", 20, "cpu_work")] + ) + ], + awaited_by=[] + ) + + # get_async_stack_trace only returns the running task + awaited_info_list = [MockAwaitedInfo(thread_id=100, awaited_by=[running_task])] + + collector = PstatsCollector(sample_interval_usec=1000) + collector.collect(awaited_info_list) + collector.create_stats() + + # Only running task should be visible + running_key = ("runner.py", 20, "cpu_work") + self.assertIn(running_key, collector.stats) + + # Verify we don't see the sleeping task (it wasn't in the input) + sleeping_key = ("sleeper.py", 10, "sleep_work") + self.assertNotIn(sleeping_key, collector.stats) + + # Task marker for running task should be present + task_keys = [k for k in collector.stats if k[0] == ""] + self.assertGreater(len(task_keys), 0, "Should have markers in stats") + + task_names = [k[2] for k in task_keys] + self.assertTrue( + any("RunningTask" in name for name in task_names), + "RunningTask should be in task markers" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_cli.py b/Lib/test/test_profiling/test_sampling_profiler/test_cli.py index 673e1c0d93c79f..e1892ec9155940 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_cli.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_cli.py @@ -547,3 +547,165 @@ def test_sort_options(self): mock_sample.assert_called_once() mock_sample.reset_mock() + + def test_async_aware_flag_defaults_to_running(self): + """Test --async-aware flag enables async profiling with default 'running' mode.""" + test_args = ["profiling.sampling.cli", "attach", "12345", "--async-aware"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.cli.sample") as mock_sample, + ): + from profiling.sampling.cli import main + main() + + mock_sample.assert_called_once() + # Verify async_aware was passed with default "running" mode + call_kwargs = mock_sample.call_args[1] + self.assertEqual(call_kwargs.get("async_aware"), "running") + + def test_async_aware_with_async_mode_all(self): + """Test --async-aware with --async-mode all.""" + test_args = ["profiling.sampling.cli", "attach", "12345", "--async-aware", "--async-mode", "all"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.cli.sample") as mock_sample, + ): + from profiling.sampling.cli import main + main() + + mock_sample.assert_called_once() + call_kwargs = mock_sample.call_args[1] + self.assertEqual(call_kwargs.get("async_aware"), "all") + + def test_async_aware_default_is_none(self): + """Test async_aware defaults to None when --async-aware not specified.""" + test_args = ["profiling.sampling.cli", "attach", "12345"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.cli.sample") as mock_sample, + ): + from profiling.sampling.cli import main + main() + + mock_sample.assert_called_once() + call_kwargs = mock_sample.call_args[1] + self.assertIsNone(call_kwargs.get("async_aware")) + + def test_async_mode_invalid_choice(self): + """Test --async-mode with invalid choice raises error.""" + test_args = ["profiling.sampling.cli", "attach", "12345", "--async-aware", "--async-mode", "invalid"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()), + self.assertRaises(SystemExit) as cm, + ): + from profiling.sampling.cli import main + main() + + self.assertEqual(cm.exception.code, 2) # argparse error + + def test_async_mode_requires_async_aware(self): + """Test --async-mode without --async-aware raises error.""" + test_args = ["profiling.sampling.cli", "attach", "12345", "--async-mode", "all"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + self.assertRaises(SystemExit) as cm, + ): + from profiling.sampling.cli import main + main() + + self.assertEqual(cm.exception.code, 2) # argparse error + error_msg = mock_stderr.getvalue() + self.assertIn("--async-mode requires --async-aware", error_msg) + + def test_async_aware_incompatible_with_native(self): + """Test --async-aware is incompatible with --native.""" + test_args = ["profiling.sampling.cli", "attach", "12345", "--async-aware", "--native"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + self.assertRaises(SystemExit) as cm, + ): + from profiling.sampling.cli import main + main() + + self.assertEqual(cm.exception.code, 2) # argparse error + error_msg = mock_stderr.getvalue() + self.assertIn("--native", error_msg) + self.assertIn("incompatible with --async-aware", error_msg) + + def test_async_aware_incompatible_with_no_gc(self): + """Test --async-aware is incompatible with --no-gc.""" + test_args = ["profiling.sampling.cli", "attach", "12345", "--async-aware", "--no-gc"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + self.assertRaises(SystemExit) as cm, + ): + from profiling.sampling.cli import main + main() + + self.assertEqual(cm.exception.code, 2) # argparse error + error_msg = mock_stderr.getvalue() + self.assertIn("--no-gc", error_msg) + self.assertIn("incompatible with --async-aware", error_msg) + + def test_async_aware_incompatible_with_both_native_and_no_gc(self): + """Test --async-aware is incompatible with both --native and --no-gc.""" + test_args = ["profiling.sampling.cli", "attach", "12345", "--async-aware", "--native", "--no-gc"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + self.assertRaises(SystemExit) as cm, + ): + from profiling.sampling.cli import main + main() + + self.assertEqual(cm.exception.code, 2) # argparse error + error_msg = mock_stderr.getvalue() + self.assertIn("--native", error_msg) + self.assertIn("--no-gc", error_msg) + self.assertIn("incompatible with --async-aware", error_msg) + + def test_async_aware_incompatible_with_mode(self): + """Test --async-aware is incompatible with --mode (non-wall).""" + test_args = ["profiling.sampling.cli", "attach", "12345", "--async-aware", "--mode", "cpu"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + self.assertRaises(SystemExit) as cm, + ): + from profiling.sampling.cli import main + main() + + self.assertEqual(cm.exception.code, 2) # argparse error + error_msg = mock_stderr.getvalue() + self.assertIn("--mode=cpu", error_msg) + self.assertIn("incompatible with --async-aware", error_msg) + + def test_async_aware_incompatible_with_all_threads(self): + """Test --async-aware is incompatible with --all-threads.""" + test_args = ["profiling.sampling.cli", "attach", "12345", "--async-aware", "--all-threads"] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + self.assertRaises(SystemExit) as cm, + ): + from profiling.sampling.cli import main + main() + + self.assertEqual(cm.exception.code, 2) # argparse error + error_msg = mock_stderr.getvalue() + self.assertIn("--all-threads", error_msg) + self.assertIn("incompatible with --async-aware", error_msg) diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_integration.py b/Lib/test/test_profiling/test_sampling_profiler/test_integration.py index e4c5032425ddcd..aae241a3335e37 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_integration.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_integration.py @@ -780,3 +780,128 @@ def test_live_incompatible_with_pstats_default_values(self): from profiling.sampling.cli import main main() self.assertNotEqual(cm.exception.code, 0) + + +@requires_subprocess() +@skip_if_not_supported +@unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", +) +class TestAsyncAwareProfilingIntegration(unittest.TestCase): + """Integration tests for async-aware profiling mode.""" + + @classmethod + def setUpClass(cls): + cls.async_script = ''' +import asyncio + +async def sleeping_leaf(): + """Leaf task that just sleeps - visible in 'all' mode.""" + for _ in range(50): + await asyncio.sleep(0.02) + +async def cpu_leaf(): + """Leaf task that does CPU work - visible in both modes.""" + total = 0 + for _ in range(200): + for i in range(10000): + total += i * i + await asyncio.sleep(0) + return total + +async def supervisor(): + """Middle layer that spawns leaf tasks.""" + tasks = [ + asyncio.create_task(sleeping_leaf(), name="Sleeper-0"), + asyncio.create_task(sleeping_leaf(), name="Sleeper-1"), + asyncio.create_task(sleeping_leaf(), name="Sleeper-2"), + asyncio.create_task(cpu_leaf(), name="Worker"), + ] + await asyncio.gather(*tasks) + +async def main(): + await supervisor() + +if __name__ == "__main__": + asyncio.run(main()) +''' + + def _collect_async_samples(self, async_aware_mode): + """Helper to collect samples and count function occurrences. + + Returns a dict mapping function names to their sample counts. + """ + with test_subprocess(self.async_script) as subproc: + try: + collector = CollapsedStackCollector(1000, skip_idle=False) + profiling.sampling.sample.sample( + subproc.process.pid, + collector, + duration_sec=SHORT_TIMEOUT, + async_aware=async_aware_mode, + ) + except PermissionError: + self.skipTest("Insufficient permissions for remote profiling") + + # Count samples per function from collapsed stacks + # stack_counter keys are (call_tree, thread_id) where call_tree + # is a tuple of (file, line, func) tuples + func_samples = {} + total = 0 + for (call_tree, _thread_id), count in collector.stack_counter.items(): + total += count + for _file, _line, func in call_tree: + func_samples[func] = func_samples.get(func, 0) + count + + func_samples["_total"] = total + return func_samples + + def test_async_aware_all_sees_sleeping_and_running_tasks(self): + """Test that async_aware='all' captures both sleeping and CPU-running tasks. + + Task tree structure: + main + └── supervisor + ├── Sleeper-0 (sleeping_leaf) + ├── Sleeper-1 (sleeping_leaf) + ├── Sleeper-2 (sleeping_leaf) + └── Worker (cpu_leaf) + + async_aware='all' should see ALL 4 leaf tasks in the output. + """ + samples = self._collect_async_samples("all") + + self.assertGreater(samples["_total"], 0, "Should have collected samples") + self.assertIn("sleeping_leaf", samples) + self.assertIn("cpu_leaf", samples) + self.assertIn("supervisor", samples) + + def test_async_aware_running_sees_only_cpu_task(self): + """Test that async_aware='running' only captures the actively running task. + + Task tree structure: + main + └── supervisor + ├── Sleeper-0 (sleeping_leaf) - NOT visible in 'running' + ├── Sleeper-1 (sleeping_leaf) - NOT visible in 'running' + ├── Sleeper-2 (sleeping_leaf) - NOT visible in 'running' + └── Worker (cpu_leaf) - VISIBLE in 'running' + + async_aware='running' should only see the Worker task doing CPU work. + """ + samples = self._collect_async_samples("running") + + total = samples["_total"] + cpu_leaf_samples = samples.get("cpu_leaf", 0) + + self.assertGreater(total, 0, "Should have collected some samples") + self.assertGreater(cpu_leaf_samples, 0, "cpu_leaf should appear in samples") + + # cpu_leaf should have at least 90% of samples (typically 99%+) + # sleeping_leaf may occasionally appear with very few samples (< 1%) + # when tasks briefly wake up to check sleep timers + cpu_percentage = (cpu_leaf_samples / total) * 100 + self.assertGreater(cpu_percentage, 90.0, + f"cpu_leaf should dominate samples in 'running' mode, " + f"got {cpu_percentage:.1f}% ({cpu_leaf_samples}/{total})") diff --git a/Misc/NEWS.d/next/Library/2025-11-14-18-00-41.gh-issue-141565.Ap2bhJ.rst b/Misc/NEWS.d/next/Library/2025-11-14-18-00-41.gh-issue-141565.Ap2bhJ.rst new file mode 100644 index 00000000000000..628f1e0af033c7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-14-18-00-41.gh-issue-141565.Ap2bhJ.rst @@ -0,0 +1 @@ +Add async-aware profiling to the Tachyon sampling profiler. The profiler now reconstructs and displays async task hierarchies in flamegraphs, making the output more actionable for users. Patch by Savannah Ostrowski and Pablo Galindo Salgado. diff --git a/Modules/_remote_debugging/_remote_debugging.h b/Modules/_remote_debugging/_remote_debugging.h index c4547baf96746b..70e362ccada6a0 100644 --- a/Modules/_remote_debugging/_remote_debugging.h +++ b/Modules/_remote_debugging/_remote_debugging.h @@ -405,6 +405,7 @@ extern PyObject* unwind_stack_for_thread( extern uintptr_t _Py_RemoteDebug_GetAsyncioDebugAddress(proc_handle_t* handle); extern int read_async_debug(RemoteUnwinderObject *unwinder); +extern int ensure_async_debug_offsets(RemoteUnwinderObject *unwinder); /* Task parsing */ extern PyObject *parse_task_name(RemoteUnwinderObject *unwinder, uintptr_t task_address); diff --git a/Modules/_remote_debugging/asyncio.c b/Modules/_remote_debugging/asyncio.c index 8552311b7dc8a0..7f91f16e3a2ce6 100644 --- a/Modules/_remote_debugging/asyncio.c +++ b/Modules/_remote_debugging/asyncio.c @@ -71,6 +71,28 @@ read_async_debug(RemoteUnwinderObject *unwinder) return result; } +int +ensure_async_debug_offsets(RemoteUnwinderObject *unwinder) +{ + // If already available, nothing to do + if (unwinder->async_debug_offsets_available) { + return 0; + } + + // Try to load async debug offsets (the target process may have + // loaded asyncio since we last checked) + if (read_async_debug(unwinder) < 0) { + PyErr_Clear(); + PyErr_SetString(PyExc_RuntimeError, "AsyncioDebug section not available"); + set_exception_cause(unwinder, PyExc_RuntimeError, + "AsyncioDebug section unavailable - asyncio module may not be loaded in target process"); + return -1; + } + + unwinder->async_debug_offsets_available = 1; + return 0; +} + /* ============================================================================ * SET ITERATION FUNCTIONS * ============================================================================ */ diff --git a/Modules/_remote_debugging/module.c b/Modules/_remote_debugging/module.c index 252291f916290c..6cd9fad37defc7 100644 --- a/Modules/_remote_debugging/module.c +++ b/Modules/_remote_debugging/module.c @@ -645,9 +645,7 @@ static PyObject * _remote_debugging_RemoteUnwinder_get_all_awaited_by_impl(RemoteUnwinderObject *self) /*[clinic end generated code: output=6a49cd345e8aec53 input=307f754cbe38250c]*/ { - if (!self->async_debug_offsets_available) { - PyErr_SetString(PyExc_RuntimeError, "AsyncioDebug section not available"); - set_exception_cause(self, PyExc_RuntimeError, "AsyncioDebug section unavailable in get_all_awaited_by"); + if (ensure_async_debug_offsets(self) < 0) { return NULL; } @@ -736,9 +734,7 @@ static PyObject * _remote_debugging_RemoteUnwinder_get_async_stack_trace_impl(RemoteUnwinderObject *self) /*[clinic end generated code: output=6433d52b55e87bbe input=6129b7d509a887c9]*/ { - if (!self->async_debug_offsets_available) { - PyErr_SetString(PyExc_RuntimeError, "AsyncioDebug section not available"); - set_exception_cause(self, PyExc_RuntimeError, "AsyncioDebug section unavailable in get_async_stack_trace"); + if (ensure_async_debug_offsets(self) < 0) { return NULL; } From 14715e3a64a674629c781d4a3dd11143ba010990 Mon Sep 17 00:00:00 2001 From: Kaisheng Xu Date: Sun, 7 Dec 2025 03:33:25 +0800 Subject: [PATCH 280/638] gh-105836: Fix `asyncio.run_coroutine_threadsafe` leaving underlying cancelled asyncio task running (#141696) Co-authored-by: Kumar Aditya --- Lib/asyncio/futures.py | 4 ++-- Lib/test/test_asyncio/test_tasks.py | 24 +++++++++++++++++++ Misc/ACKS | 1 + ...-11-18-15-48-13.gh-issue-105836.sbUw24.rst | 2 ++ 4 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-18-15-48-13.gh-issue-105836.sbUw24.rst diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index 6bd00a644789f1..29652295218a22 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -389,7 +389,7 @@ def _set_state(future, other): def _call_check_cancel(destination): if destination.cancelled(): - if source_loop is None or source_loop is dest_loop: + if source_loop is None or source_loop is events._get_running_loop(): source.cancel() else: source_loop.call_soon_threadsafe(source.cancel) @@ -398,7 +398,7 @@ def _call_set_state(source): if (destination.cancelled() and dest_loop is not None and dest_loop.is_closed()): return - if dest_loop is None or dest_loop is source_loop: + if dest_loop is None or dest_loop is events._get_running_loop(): _set_state(destination, source) else: if dest_loop.is_closed(): diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 931a43816a257a..9809621a324450 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -3680,6 +3680,30 @@ def task_factory(loop, coro): (loop, context), kwargs = callback.call_args self.assertEqual(context['exception'], exc_context.exception) + def test_run_coroutine_threadsafe_and_cancel(self): + task = None + thread_future = None + # Use a custom task factory to capture the created Task + def task_factory(loop, coro): + nonlocal task + task = asyncio.Task(coro, loop=loop) + return task + + self.addCleanup(self.loop.set_task_factory, + self.loop.get_task_factory()) + + async def target(): + nonlocal thread_future + self.loop.set_task_factory(task_factory) + thread_future = asyncio.run_coroutine_threadsafe(asyncio.sleep(10), self.loop) + await asyncio.sleep(0) + + thread_future.cancel() + + self.loop.run_until_complete(target()) + self.assertTrue(task.cancelled()) + self.assertTrue(thread_future.cancelled()) + class SleepTests(test_utils.TestCase): def setUp(self): diff --git a/Misc/ACKS b/Misc/ACKS index ab6b8662d8fc81..e3927ff0b3364e 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -2119,6 +2119,7 @@ Xiang Zhang Robert Xiao Florent Xicluna Yanbo, Xie +Kaisheng Xu Xinhang Xu Arnon Yaari Alakshendra Yadav diff --git a/Misc/NEWS.d/next/Library/2025-11-18-15-48-13.gh-issue-105836.sbUw24.rst b/Misc/NEWS.d/next/Library/2025-11-18-15-48-13.gh-issue-105836.sbUw24.rst new file mode 100644 index 00000000000000..d2edc5b2cb743d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-18-15-48-13.gh-issue-105836.sbUw24.rst @@ -0,0 +1,2 @@ +Fix :meth:`asyncio.run_coroutine_threadsafe` leaving underlying cancelled +asyncio task running. From c91c373ef6ceab78936d866572023feb642acc73 Mon Sep 17 00:00:00 2001 From: ivonastojanovic <80911834+ivonastojanovic@users.noreply.github.com> Date: Sat, 6 Dec 2025 20:27:16 +0000 Subject: [PATCH 281/638] gh-140677 Improve heatmap colors (#142241) Co-authored-by: Pablo Galindo Salgado --- .../sampling/_heatmap_assets/heatmap.css | 20 ++- .../sampling/_heatmap_assets/heatmap.js | 49 +++--- .../sampling/_heatmap_assets/heatmap_index.js | 16 ++ .../_heatmap_assets/heatmap_shared.js | 40 +++++ Lib/profiling/sampling/heatmap_collector.py | 143 +++--------------- Lib/test/test_profiling/test_heatmap.py | 6 - 6 files changed, 117 insertions(+), 157 deletions(-) create mode 100644 Lib/profiling/sampling/_heatmap_assets/heatmap_shared.js diff --git a/Lib/profiling/sampling/_heatmap_assets/heatmap.css b/Lib/profiling/sampling/_heatmap_assets/heatmap.css index 44915b2a2da7b8..ada6d2f2ee1db6 100644 --- a/Lib/profiling/sampling/_heatmap_assets/heatmap.css +++ b/Lib/profiling/sampling/_heatmap_assets/heatmap.css @@ -1094,18 +1094,34 @@ } #scroll_marker .marker.cold { + background: var(--heat-1); +} + +#scroll_marker .marker.cool { background: var(--heat-2); } +#scroll_marker .marker.mild { + background: var(--heat-3); +} + #scroll_marker .marker.warm { - background: var(--heat-5); + background: var(--heat-4); } #scroll_marker .marker.hot { + background: var(--heat-5); +} + +#scroll_marker .marker.very-hot { + background: var(--heat-6); +} + +#scroll_marker .marker.intense { background: var(--heat-7); } -#scroll_marker .marker.vhot { +#scroll_marker .marker.extreme { background: var(--heat-8); } diff --git a/Lib/profiling/sampling/_heatmap_assets/heatmap.js b/Lib/profiling/sampling/_heatmap_assets/heatmap.js index ccf823863638dd..5a7ff5dd61ad3a 100644 --- a/Lib/profiling/sampling/_heatmap_assets/heatmap.js +++ b/Lib/profiling/sampling/_heatmap_assets/heatmap.js @@ -26,6 +26,7 @@ function toggleTheme() { if (btn) { btn.innerHTML = next === 'dark' ? '☼' : '☾'; // sun or moon } + applyLineColors(); // Rebuild scroll marker with new theme colors buildScrollMarker(); @@ -160,13 +161,6 @@ function getSampleCount(line) { return parseInt(text) || 0; } -function getIntensityClass(ratio) { - if (ratio > 0.75) return 'vhot'; - if (ratio > 0.5) return 'hot'; - if (ratio > 0.25) return 'warm'; - return 'cold'; -} - // ============================================================================ // Scroll Minimap // ============================================================================ @@ -194,7 +188,7 @@ function buildScrollMarker() { const lineTop = Math.floor(line.offsetTop * markerScale); const lineNumber = index + 1; - const intensityClass = maxSamples > 0 ? getIntensityClass(samples / maxSamples) : 'cold'; + const intensityClass = maxSamples > 0 ? (intensityToClass(samples / maxSamples) || 'cold') : 'cold'; if (lineNumber === prevLine + 1 && lastMark?.classList.contains(intensityClass)) { lastMark.style.height = `${lineTop + lineHeight - lastTop}px`; @@ -212,6 +206,21 @@ function buildScrollMarker() { document.body.appendChild(scrollMarker); } +function applyLineColors() { + const lines = document.querySelectorAll('.code-line'); + lines.forEach(line => { + let intensity; + if (colorMode === 'self') { + intensity = parseFloat(line.getAttribute('data-self-intensity')) || 0; + } else { + intensity = parseFloat(line.getAttribute('data-cumulative-intensity')) || 0; + } + + const color = intensityToColor(intensity); + line.style.background = color; + }); +} + // ============================================================================ // Toggle Controls // ============================================================================ @@ -264,20 +273,7 @@ function applyHotFilter() { function toggleColorMode() { colorMode = colorMode === 'self' ? 'cumulative' : 'self'; - const lines = document.querySelectorAll('.code-line'); - - lines.forEach(line => { - let bgColor; - if (colorMode === 'self') { - bgColor = line.getAttribute('data-self-color'); - } else { - bgColor = line.getAttribute('data-cumulative-color'); - } - - if (bgColor) { - line.style.background = bgColor; - } - }); + applyLineColors(); updateToggleUI('toggle-color-mode', colorMode === 'cumulative'); @@ -295,14 +291,7 @@ function toggleColorMode() { document.addEventListener('DOMContentLoaded', function() { // Restore UI state (theme, etc.) restoreUIState(); - - // Apply background colors - document.querySelectorAll('.code-line[data-bg-color]').forEach(line => { - const bgColor = line.getAttribute('data-bg-color'); - if (bgColor) { - line.style.background = bgColor; - } - }); + applyLineColors(); // Initialize navigation buttons document.querySelectorAll('.nav-btn').forEach(button => { diff --git a/Lib/profiling/sampling/_heatmap_assets/heatmap_index.js b/Lib/profiling/sampling/_heatmap_assets/heatmap_index.js index 5f3e65c3310884..4ddacca5173d34 100644 --- a/Lib/profiling/sampling/_heatmap_assets/heatmap_index.js +++ b/Lib/profiling/sampling/_heatmap_assets/heatmap_index.js @@ -1,6 +1,19 @@ // Tachyon Profiler - Heatmap Index JavaScript // Index page specific functionality +// ============================================================================ +// Heatmap Bar Coloring +// ============================================================================ + +function applyHeatmapBarColors() { + const bars = document.querySelectorAll('.heatmap-bar[data-intensity]'); + bars.forEach(bar => { + const intensity = parseFloat(bar.getAttribute('data-intensity')) || 0; + const color = intensityToColor(intensity); + bar.style.backgroundColor = color; + }); +} + // ============================================================================ // Theme Support // ============================================================================ @@ -17,6 +30,8 @@ function toggleTheme() { if (btn) { btn.innerHTML = next === 'dark' ? '☼' : '☾'; // sun or moon } + + applyHeatmapBarColors(); } function restoreUIState() { @@ -108,4 +123,5 @@ function collapseAll() { document.addEventListener('DOMContentLoaded', function() { restoreUIState(); + applyHeatmapBarColors(); }); diff --git a/Lib/profiling/sampling/_heatmap_assets/heatmap_shared.js b/Lib/profiling/sampling/_heatmap_assets/heatmap_shared.js new file mode 100644 index 00000000000000..f44ebcff4ffe89 --- /dev/null +++ b/Lib/profiling/sampling/_heatmap_assets/heatmap_shared.js @@ -0,0 +1,40 @@ +// Tachyon Profiler - Shared Heatmap JavaScript +// Common utilities shared between index and file views + +// ============================================================================ +// Heat Level Mapping (Single source of truth for intensity thresholds) +// ============================================================================ + +// Maps intensity (0-1) to heat level (0-8). Level 0 = no heat, 1-8 = heat levels. +function intensityToHeatLevel(intensity) { + if (intensity <= 0) return 0; + if (intensity <= 0.125) return 1; + if (intensity <= 0.25) return 2; + if (intensity <= 0.375) return 3; + if (intensity <= 0.5) return 4; + if (intensity <= 0.625) return 5; + if (intensity <= 0.75) return 6; + if (intensity <= 0.875) return 7; + return 8; +} + +// Class names corresponding to heat levels 1-8 (used by scroll marker) +const HEAT_CLASS_NAMES = ['cold', 'cool', 'mild', 'warm', 'hot', 'very-hot', 'intense', 'extreme']; + +function intensityToClass(intensity) { + const level = intensityToHeatLevel(intensity); + return level === 0 ? null : HEAT_CLASS_NAMES[level - 1]; +} + +// ============================================================================ +// Color Mapping (Intensity to Heat Color) +// ============================================================================ + +function intensityToColor(intensity) { + const level = intensityToHeatLevel(intensity); + if (level === 0) { + return 'transparent'; + } + const rootStyle = getComputedStyle(document.documentElement); + return rootStyle.getPropertyValue(`--heat-${level}`).trim(); +} diff --git a/Lib/profiling/sampling/heatmap_collector.py b/Lib/profiling/sampling/heatmap_collector.py index eb128aba9b197f..8a8ba9628df573 100644 --- a/Lib/profiling/sampling/heatmap_collector.py +++ b/Lib/profiling/sampling/heatmap_collector.py @@ -5,6 +5,7 @@ import html import importlib.resources import json +import math import os import platform import site @@ -44,31 +45,6 @@ class TreeNode: children: Dict[str, 'TreeNode'] = field(default_factory=dict) -@dataclass -class ColorGradient: - """Configuration for heatmap color gradient calculations.""" - # Color stops thresholds - stop_1: float = 0.2 # Blue to cyan transition - stop_2: float = 0.4 # Cyan to green transition - stop_3: float = 0.6 # Green to yellow transition - stop_4: float = 0.8 # Yellow to orange transition - stop_5: float = 1.0 # Orange to red transition - - # Alpha (opacity) values - alpha_very_cold: float = 0.3 - alpha_cold: float = 0.4 - alpha_medium: float = 0.5 - alpha_warm: float = 0.6 - alpha_hot_base: float = 0.7 - alpha_hot_range: float = 0.15 - - # Gradient multiplier - multiplier: int = 5 - - # Cache for calculated colors - cache: Dict[float, Tuple[int, int, int, float]] = field(default_factory=dict) - - # ============================================================================ # Module Path Analysis # ============================================================================ @@ -224,8 +200,9 @@ def _load_templates(self): self.file_css = css_content # Load JS - self.index_js = (assets_dir / "heatmap_index.js").read_text(encoding="utf-8") - self.file_js = (assets_dir / "heatmap.js").read_text(encoding="utf-8") + shared_js = (assets_dir / "heatmap_shared.js").read_text(encoding="utf-8") + self.index_js = f"{shared_js}\n{(assets_dir / 'heatmap_index.js').read_text(encoding='utf-8')}" + self.file_js = f"{shared_js}\n{(assets_dir / 'heatmap.js').read_text(encoding='utf-8')}" # Load Python logo logo_dir = template_dir / "_assets" @@ -321,18 +298,13 @@ def _calculate_node_stats(node: TreeNode) -> Tuple[int, int]: class _HtmlRenderer: """Renders hierarchical tree structures as HTML.""" - def __init__(self, file_index: Dict[str, str], color_gradient: ColorGradient, - calculate_intensity_color_func): - """Initialize renderer with file index and color calculation function. + def __init__(self, file_index: Dict[str, str]): + """Initialize renderer with file index. Args: file_index: Mapping from filenames to HTML file names - color_gradient: ColorGradient configuration - calculate_intensity_color_func: Function to calculate colors """ self.file_index = file_index - self.color_gradient = color_gradient - self.calculate_intensity_color = calculate_intensity_color_func self.heatmap_bar_height = 16 def render_hierarchical_html(self, trees: Dict[str, TreeNode]) -> str: @@ -450,8 +422,6 @@ def _render_file_item(self, stat: FileStats, indent: str = '') -> str: module_name = html.escape(stat.module_name) intensity = stat.percentage / 100.0 - r, g, b, alpha = self.calculate_intensity_color(intensity) - bg_color = f"rgba({r}, {g}, {b}, {alpha})" bar_width = min(stat.percentage, 100) html_file = self.file_index[stat.filename] @@ -459,7 +429,7 @@ def _render_file_item(self, stat: FileStats, indent: str = '') -> str: return (f'{indent}
\n' f'{indent} 📄 {module_name}\n' f'{indent} {stat.total_samples:,} samples\n' - f'{indent}
\n' + f'{indent}
\n' f'{indent}
\n') @@ -501,20 +471,12 @@ def __init__(self, *args, **kwargs): self._path_info = get_python_path_info() self.stats = {} - # Color gradient configuration - self._color_gradient = ColorGradient() - # Template loader (loads all templates once) self._template_loader = _TemplateLoader() # File index (populated during export) self.file_index = {} - @property - def _color_cache(self): - """Compatibility property for accessing color cache.""" - return self._color_gradient.cache - def set_stats(self, sample_interval_usec, duration_sec, sample_rate, error_rate=None, missed_samples=None, **kwargs): """Set profiling statistics to include in heatmap output. @@ -746,8 +708,7 @@ def _generate_index_html(self, index_path: Path, file_stats: List[FileStats]): tree = _TreeBuilder.build_file_tree(file_stats) # Render tree as HTML - renderer = _HtmlRenderer(self.file_index, self._color_gradient, - self._calculate_intensity_color) + renderer = _HtmlRenderer(self.file_index) sections_html = renderer.render_hierarchical_html(tree) # Format error rate and missed samples with bar classes @@ -809,56 +770,6 @@ def _generate_index_html(self, index_path: Path, file_stats: List[FileStats]): except (IOError, OSError) as e: raise RuntimeError(f"Failed to write index file {index_path}: {e}") from e - def _calculate_intensity_color(self, intensity: float) -> Tuple[int, int, int, float]: - """Calculate RGB color and alpha for given intensity (0-1 range). - - Returns (r, g, b, alpha) tuple representing the heatmap color gradient: - blue -> green -> yellow -> orange -> red - - Results are cached to improve performance. - """ - # Round to 3 decimal places for cache key - cache_key = round(intensity, 3) - if cache_key in self._color_gradient.cache: - return self._color_gradient.cache[cache_key] - - gradient = self._color_gradient - m = gradient.multiplier - - # Color stops with (threshold, rgb_func, alpha_func) - stops = [ - (gradient.stop_1, - lambda i: (0, int(150 * i * m), 255), - lambda i: gradient.alpha_very_cold), - (gradient.stop_2, - lambda i: (0, 255, int(255 * (1 - (i - gradient.stop_1) * m))), - lambda i: gradient.alpha_cold), - (gradient.stop_3, - lambda i: (int(255 * (i - gradient.stop_2) * m), 255, 0), - lambda i: gradient.alpha_medium), - (gradient.stop_4, - lambda i: (255, int(200 - 100 * (i - gradient.stop_3) * m), 0), - lambda i: gradient.alpha_warm), - (gradient.stop_5, - lambda i: (255, int(100 * (1 - (i - gradient.stop_4) * m)), 0), - lambda i: gradient.alpha_hot_base + gradient.alpha_hot_range * (i - gradient.stop_4) * m), - ] - - result = None - for threshold, rgb_func, alpha_func in stops: - if intensity < threshold or threshold == gradient.stop_5: - r, g, b = rgb_func(intensity) - result = (r, g, b, alpha_func(intensity)) - break - - # Fallback - if result is None: - result = (255, 0, 0, 0.75) - - # Cache the result - self._color_gradient.cache[cache_key] = result - return result - def _generate_file_html(self, output_path: Path, filename: str, line_counts: Dict[int, int], self_counts: Dict[int, int], file_stat: FileStats): @@ -913,25 +824,23 @@ def _build_line_html(self, line_num: int, line_content: str, # Calculate colors for both self and cumulative modes if cumulative_samples > 0: - cumulative_intensity = cumulative_samples / max_samples if max_samples > 0 else 0 - self_intensity = self_samples / max_self_samples if max_self_samples > 0 and self_samples > 0 else 0 - - # Default to self-based coloring - intensity = self_intensity if self_samples > 0 else cumulative_intensity - r, g, b, alpha = self._calculate_intensity_color(intensity) - bg_color = f"rgba({r}, {g}, {b}, {alpha})" - - # Pre-calculate colors for both modes (for JS toggle) - self_bg_color = self._format_color_for_intensity(self_intensity) if self_samples > 0 else "transparent" - cumulative_bg_color = self._format_color_for_intensity(cumulative_intensity) + log_cumulative = math.log(cumulative_samples + 1) + log_max = math.log(max_samples + 1) + cumulative_intensity = log_cumulative / log_max if log_max > 0 else 0 + + if self_samples > 0 and max_self_samples > 0: + log_self = math.log(self_samples + 1) + log_max_self = math.log(max_self_samples + 1) + self_intensity = log_self / log_max_self if log_max_self > 0 else 0 + else: + self_intensity = 0 self_display = f"{self_samples:,}" if self_samples > 0 else "" cumulative_display = f"{cumulative_samples:,}" tooltip = f"Self: {self_samples:,}, Total: {cumulative_samples:,}" else: - bg_color = "transparent" - self_bg_color = "transparent" - cumulative_bg_color = "transparent" + cumulative_intensity = 0 + self_intensity = 0 self_display = "" cumulative_display = "" tooltip = "" @@ -939,13 +848,14 @@ def _build_line_html(self, line_num: int, line_content: str, # Get navigation buttons nav_buttons_html = self._build_navigation_buttons(filename, line_num) - # Build line HTML + # Build line HTML with intensity data attributes line_html = html.escape(line_content.rstrip('\n')) title_attr = f' title="{html.escape(tooltip)}"' if tooltip else "" return ( - f'
\n' f'
{line_num}
\n' f'
{self_display}
\n' @@ -955,11 +865,6 @@ def _build_line_html(self, line_num: int, line_content: str, f'
\n' ) - def _format_color_for_intensity(self, intensity: float) -> str: - """Format color as rgba() string for given intensity.""" - r, g, b, alpha = self._calculate_intensity_color(intensity) - return f"rgba({r}, {g}, {b}, {alpha})" - def _build_navigation_buttons(self, filename: str, line_num: int) -> str: """Build navigation buttons for callers/callees.""" line_key = (filename, line_num) diff --git a/Lib/test/test_profiling/test_heatmap.py b/Lib/test/test_profiling/test_heatmap.py index a6ff3b83ea1e0b..24bf3d21c2fa04 100644 --- a/Lib/test/test_profiling/test_heatmap.py +++ b/Lib/test/test_profiling/test_heatmap.py @@ -147,12 +147,6 @@ def test_init_sets_total_samples_to_zero(self): collector = HeatmapCollector(sample_interval_usec=100) self.assertEqual(collector._total_samples, 0) - def test_init_creates_color_cache(self): - """Test that color cache is initialized.""" - collector = HeatmapCollector(sample_interval_usec=100) - self.assertIsInstance(collector._color_cache, dict) - self.assertEqual(len(collector._color_cache), 0) - def test_init_gets_path_info(self): """Test that path info is retrieved during init.""" collector = HeatmapCollector(sample_interval_usec=100) From 100e316e53abfff45f2a94987ee7a8622fcd3589 Mon Sep 17 00:00:00 2001 From: Sanyam Khurana <8039608+CuriousLearner@users.noreply.github.com> Date: Sat, 6 Dec 2025 15:47:08 -0500 Subject: [PATCH 282/638] gh-69113: Fix doctest to report line numbers for __test__ strings (#141624) Enhanced the _find_lineno method in doctest to correctly identify and report line numbers for doctests defined in __test__ dictionaries when formatted as triple-quoted strings. Finds a non-blank line in the test string and matches it in the source file, verifying subsequent lines also match to handle duplicate lines. Previously, doctest would report "line None" for __test__ dictionary strings, making it difficult to debug failing tests. Co-authored-by: Jurjen N.E. Bos Co-authored-by: R. David Murray --- Lib/doctest.py | 26 ++++ Lib/test/test_doctest/test_doctest.py | 122 +++++++++++++++++- ...5-11-16-04-40-06.gh-issue-69113.Xy7Fmn.rst | 1 + 3 files changed, 145 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-16-04-40-06.gh-issue-69113.Xy7Fmn.rst diff --git a/Lib/doctest.py b/Lib/doctest.py index ad8fb900f692c7..0fcfa1e3e97144 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -1167,6 +1167,32 @@ def _find_lineno(self, obj, source_lines): if pat.match(source_lines[lineno]): return lineno + # Handle __test__ string doctests formatted as triple-quoted + # strings. Find a non-blank line in the test string and match it + # in the source, verifying subsequent lines also match to handle + # duplicate lines. + if isinstance(obj, str) and source_lines is not None: + obj_lines = obj.splitlines(keepends=True) + # Skip the first line (may be on same line as opening quotes) + # and any blank lines to find a meaningful line to match. + start_index = 1 + while (start_index < len(obj_lines) + and not obj_lines[start_index].strip()): + start_index += 1 + if start_index < len(obj_lines): + target_line = obj_lines[start_index] + for lineno, source_line in enumerate(source_lines): + if source_line == target_line: + # Verify subsequent lines also match + for i in range(start_index + 1, len(obj_lines) - 1): + source_idx = lineno + i - start_index + if source_idx >= len(source_lines): + break + if obj_lines[i] != source_lines[source_idx]: + break + else: + return lineno - start_index + # We couldn't find the line number. return None diff --git a/Lib/test/test_doctest/test_doctest.py b/Lib/test/test_doctest/test_doctest.py index 0fa74407e3c436..241d09db1fa70e 100644 --- a/Lib/test/test_doctest/test_doctest.py +++ b/Lib/test/test_doctest/test_doctest.py @@ -833,6 +833,118 @@ def test_empty_namespace_package(self): self.assertEqual(len(include_empty_finder.find(mod)), 1) self.assertEqual(len(exclude_empty_finder.find(mod)), 0) + def test_lineno_of_test_dict_strings(self): + """Test line numbers are found for __test__ dict strings.""" + module_content = '''\ +"""Module docstring.""" + +def dummy_function(): + """Dummy function docstring.""" + pass + +__test__ = { + 'test_string': """ + This is a test string. + >>> 1 + 1 + 2 + """, +} +''' + with tempfile.TemporaryDirectory() as tmpdir: + module_path = os.path.join(tmpdir, 'test_module_lineno.py') + with open(module_path, 'w') as f: + f.write(module_content) + + sys.path.insert(0, tmpdir) + try: + import test_module_lineno + finder = doctest.DocTestFinder() + tests = finder.find(test_module_lineno) + + test_dict_test = None + for test in tests: + if '__test__' in test.name: + test_dict_test = test + break + + self.assertIsNotNone( + test_dict_test, + "__test__ dict test not found" + ) + # gh-69113: line number should not be None for __test__ strings + self.assertIsNotNone( + test_dict_test.lineno, + "Line number should not be None for __test__ dict strings" + ) + self.assertGreater( + test_dict_test.lineno, + 0, + "Line number should be positive" + ) + finally: + if 'test_module_lineno' in sys.modules: + del sys.modules['test_module_lineno'] + sys.path.pop(0) + + def test_lineno_multiline_matching(self): + """Test multi-line matching when no unique line exists.""" + # gh-69113: test that line numbers are found even when lines + # appear multiple times (e.g., ">>> x = 1" in both test entries) + module_content = '''\ +"""Module docstring.""" + +__test__ = { + 'test_one': """ + >>> x = 1 + >>> x + 1 + """, + 'test_two': """ + >>> x = 1 + >>> x + 2 + """, +} +''' + with tempfile.TemporaryDirectory() as tmpdir: + module_path = os.path.join(tmpdir, 'test_module_multiline.py') + with open(module_path, 'w') as f: + f.write(module_content) + + sys.path.insert(0, tmpdir) + try: + import test_module_multiline + finder = doctest.DocTestFinder() + tests = finder.find(test_module_multiline) + + test_one = None + test_two = None + for test in tests: + if 'test_one' in test.name: + test_one = test + elif 'test_two' in test.name: + test_two = test + + self.assertIsNotNone(test_one, "test_one not found") + self.assertIsNotNone(test_two, "test_two not found") + self.assertIsNotNone( + test_one.lineno, + "Line number should not be None for test_one" + ) + self.assertIsNotNone( + test_two.lineno, + "Line number should not be None for test_two" + ) + self.assertNotEqual( + test_one.lineno, + test_two.lineno, + "test_one and test_two should have different line numbers" + ) + finally: + if 'test_module_multiline' in sys.modules: + del sys.modules['test_module_multiline'] + sys.path.pop(0) + def test_DocTestParser(): r""" Unit tests for the `DocTestParser` class. @@ -2434,7 +2546,8 @@ def test_DocTestSuite_errors(): >>> print(result.failures[1][1]) # doctest: +ELLIPSIS Traceback (most recent call last): - File "...sample_doctest_errors.py", line None, in test.test_doctest.sample_doctest_errors.__test__.bad + File "...sample_doctest_errors.py", line 37, in test.test_doctest.sample_doctest_errors.__test__.bad + >...>> 2 + 2 AssertionError: Failed example: 2 + 2 Expected: @@ -2464,7 +2577,8 @@ def test_DocTestSuite_errors(): >>> print(result.errors[1][1]) # doctest: +ELLIPSIS Traceback (most recent call last): - File "...sample_doctest_errors.py", line None, in test.test_doctest.sample_doctest_errors.__test__.bad + File "...sample_doctest_errors.py", line 39, in test.test_doctest.sample_doctest_errors.__test__.bad + >...>> 1/0 File "", line 1, in 1/0 ~^~ @@ -3256,7 +3370,7 @@ def test_testmod_errors(): r""" ~^~ ZeroDivisionError: division by zero ********************************************************************** - File "...sample_doctest_errors.py", line ?, in test.test_doctest.sample_doctest_errors.__test__.bad + File "...sample_doctest_errors.py", line 37, in test.test_doctest.sample_doctest_errors.__test__.bad Failed example: 2 + 2 Expected: @@ -3264,7 +3378,7 @@ def test_testmod_errors(): r""" Got: 4 ********************************************************************** - File "...sample_doctest_errors.py", line ?, in test.test_doctest.sample_doctest_errors.__test__.bad + File "...sample_doctest_errors.py", line 39, in test.test_doctest.sample_doctest_errors.__test__.bad Failed example: 1/0 Exception raised: diff --git a/Misc/NEWS.d/next/Library/2025-11-16-04-40-06.gh-issue-69113.Xy7Fmn.rst b/Misc/NEWS.d/next/Library/2025-11-16-04-40-06.gh-issue-69113.Xy7Fmn.rst new file mode 100644 index 00000000000000..cd76ae9b11ef28 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-16-04-40-06.gh-issue-69113.Xy7Fmn.rst @@ -0,0 +1 @@ +Fix :mod:`doctest` to correctly report line numbers for doctests in ``__test__`` dictionary when formatted as triple-quoted strings by finding unique lines in the string and matching them in the source file. From 07eff899d8a8ee4c4b1be7cb223fe25687f6216c Mon Sep 17 00:00:00 2001 From: Paresh Joshi Date: Sun, 7 Dec 2025 02:29:35 +0530 Subject: [PATCH 283/638] gh-142006: Fix HeaderWriteError in email.policy.default caused by extra newline (#142008) RDM: This fixes a subtle folding error that showed up when a token exactly filled a line and was followed by whitespace and a token with no folding whitespace that was longer than a line. In this particular circumstance the whitespace after the first token got pushed on to the next line, and then stolen to go in front of the next unfoldable token...leaving a completely empty line in the line buffer. That line got turned in to a newline, which is RFC illegal, and the newish security check caught it. The fix is to just delete that empty line from the buffer. Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> --- Lib/email/_header_value_parser.py | 3 +++ Lib/test/test_email/test__header_value_parser.py | 10 ++++++++++ .../2025-11-27-10-49-13.gh-issue-142006.nzJDG5.rst | 1 + 3 files changed, 14 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2025-11-27-10-49-13.gh-issue-142006.nzJDG5.rst diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index c7f665b3990512..cbff9694742490 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -2792,6 +2792,9 @@ def _steal_trailing_WSP_if_exists(lines): if lines and lines[-1] and lines[-1][-1] in WSP: wsp = lines[-1][-1] lines[-1] = lines[-1][:-1] + # gh-142006: if the line is now empty, remove it entirely. + if not lines[-1]: + lines.pop() return wsp def _refold_parse_tree(parse_tree, *, policy): diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index 179e236ecdfd7f..f7f9f9c4e2fbb5 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -3255,5 +3255,15 @@ def test_long_filename_attachment(self): " filename*1*=_TEST_TES.txt\n", ) + def test_fold_unfoldable_element_stealing_whitespace(self): + # gh-142006: When an element is too long to fit on the current line + # the previous line's trailing whitespace should not trigger a double newline. + policy = self.policy.clone(max_line_length=10) + # The non-whitespace text needs to exactly fill the max_line_length (10). + text = ("a" * 9) + ", " + ("b" * 20) + expected = ("a" * 9) + ",\n " + ("b" * 20) + "\n" + token = parser.get_address_list(text)[0] + self._test(token, expected, policy=policy) + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2025-11-27-10-49-13.gh-issue-142006.nzJDG5.rst b/Misc/NEWS.d/next/Library/2025-11-27-10-49-13.gh-issue-142006.nzJDG5.rst new file mode 100644 index 00000000000000..49643892ff9ccd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-27-10-49-13.gh-issue-142006.nzJDG5.rst @@ -0,0 +1 @@ +Fix a bug in the :mod:`email.policy.default` folding algorithm which incorrectly resulted in a doubled newline when a line ending at exactly max_line_length was followed by an unfoldable token. From ed4f78a4b318e57c9cadad2296c8b977431df6b3 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sat, 6 Dec 2025 21:09:35 +0000 Subject: [PATCH 284/638] gh-142236: Fix incorrect keyword suggestions for syntax errors (#142328) The keyword typo suggestion mechanism in traceback would incorrectly suggest replacements when the extracted source code was merely incomplete rather than containing an actual typo. For example, when a missing comma caused a syntax error, the system would suggest replacing 'print' with 'not' because the incomplete code snippet happened to pass validation. The fix adds a validation step that first checks whether the original extracted code raises a SyntaxError. If the code compiles successfully or is simply incomplete (compile_command returns None), the function returns early since there is no way to verify that a keyword replacement would actually fix the problem. --- Lib/test/test_traceback.py | 17 +++++++++++++++++ Lib/traceback.py | 9 +++++++++ ...25-12-06-00-16-43.gh-issue-142236.m3EF9E.rst | 4 ++++ 3 files changed, 30 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-12-06-00-16-43.gh-issue-142236.m3EF9E.rst diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 3876f1a74bbc1a..d107ad925941fe 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -1784,6 +1784,23 @@ def test_keyword_suggestions_from_command_string(self): stderr_text = stderr.decode('utf-8') self.assertIn(f"Did you mean '{expected_kw}'", stderr_text) + def test_no_keyword_suggestion_for_comma_errors(self): + # When the parser identifies a missing comma, don't suggest + # bogus keyword replacements like 'print' -> 'not' + code = '''\ +import sys +print( + "line1" + "line2" + file=sys.stderr +) +''' + source = textwrap.dedent(code).strip() + rc, stdout, stderr = assert_python_failure('-c', source) + stderr_text = stderr.decode('utf-8') + self.assertIn("Perhaps you forgot a comma", stderr_text) + self.assertNotIn("Did you mean", stderr_text) + @requires_debug_ranges() @force_not_colorized_test_class class PurePythonTracebackErrorCaretTests( diff --git a/Lib/traceback.py b/Lib/traceback.py index 8a3e0f77e765dc..c1052adeed25a1 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -1340,6 +1340,15 @@ def _find_keyword_typos(self): if len(error_code) > 1024: return + # If the original code doesn't raise SyntaxError, we can't validate + # that a keyword replacement actually fixes anything + try: + codeop.compile_command(error_code, symbol="exec", flags=codeop.PyCF_ONLY_AST) + except SyntaxError: + pass # Good - the original code has a syntax error we might fix + else: + return # Original code compiles or is incomplete - can't validate fixes + error_lines = error_code.splitlines() tokens = tokenize.generate_tokens(io.StringIO(error_code).readline) tokens_left_to_process = 10 diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-12-06-00-16-43.gh-issue-142236.m3EF9E.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-06-00-16-43.gh-issue-142236.m3EF9E.rst new file mode 100644 index 00000000000000..b5c6a27fd6a259 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-06-00-16-43.gh-issue-142236.m3EF9E.rst @@ -0,0 +1,4 @@ +Fix incorrect keyword suggestions for syntax errors in :mod:`traceback`. The +keyword typo suggestion mechanism would incorrectly suggest replacements when +the extracted source code was incomplete rather than containing an actual typo. +Patch by Pablo Galindo. From 9d707d8a64cd8042cd4ca9633a3feff8dee4a06d Mon Sep 17 00:00:00 2001 From: Ivo Bellin Salarin Date: Sat, 6 Dec 2025 22:54:29 +0100 Subject: [PATCH 285/638] gh-68552: fix defects policy (#138579) Extend defect handling via policy to a couple of missed defects. --------- Co-authored-by: Martin Panter Co-authored-by: Ivo Bellin Salarin --- Lib/email/feedparser.py | 7 +- Lib/test/test_email/test_defect_handling.py | 46 +++++++++- Lib/test/test_email/test_email.py | 88 ------------------- ...5-12-04-09-22-31.gh-issue-68552.I_v-xB.rst | 1 + 4 files changed, 48 insertions(+), 94 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-12-04-09-22-31.gh-issue-68552.I_v-xB.rst diff --git a/Lib/email/feedparser.py b/Lib/email/feedparser.py index 6479b9bab7ac5b..ae8ef32792b3e9 100644 --- a/Lib/email/feedparser.py +++ b/Lib/email/feedparser.py @@ -504,10 +504,9 @@ def _parse_headers(self, lines): self._input.unreadline(line) return else: - # Weirdly placed unix-from line. Note this as a defect - # and ignore it. + # Weirdly placed unix-from line. defect = errors.MisplacedEnvelopeHeaderDefect(line) - self._cur.defects.append(defect) + self.policy.handle_defect(self._cur, defect) continue # Split the line on the colon separating field name from value. # There will always be a colon, because if there wasn't the part of @@ -519,7 +518,7 @@ def _parse_headers(self, lines): # message. Track the error but keep going. if i == 0: defect = errors.InvalidHeaderDefect("Missing header name.") - self._cur.defects.append(defect) + self.policy.handle_defect(self._cur, defect) continue assert i>0, "_parse_headers fed line with no : and no leading WS" diff --git a/Lib/test/test_email/test_defect_handling.py b/Lib/test/test_email/test_defect_handling.py index 44e76c8ce5e03a..acc4accccac756 100644 --- a/Lib/test/test_email/test_defect_handling.py +++ b/Lib/test/test_email/test_defect_handling.py @@ -126,12 +126,10 @@ def test_multipart_invalid_cte(self): errors.InvalidMultipartContentTransferEncodingDefect) def test_multipart_no_cte_no_defect(self): - if self.raise_expected: return msg = self._str_msg(self.multipart_msg.format('')) self.assertEqual(len(self.get_defects(msg)), 0) def test_multipart_valid_cte_no_defect(self): - if self.raise_expected: return for cte in ('7bit', '8bit', 'BINary'): msg = self._str_msg( self.multipart_msg.format("\nContent-Transfer-Encoding: "+cte)) @@ -300,6 +298,47 @@ def test_missing_ending_boundary(self): self.assertDefectsEqual(self.get_defects(msg), [errors.CloseBoundaryNotFoundDefect]) + def test_line_beginning_colon(self): + string = ( + "Subject: Dummy subject\r\n: faulty header line\r\n\r\nbody\r\n" + ) + + with self._raise_point(errors.InvalidHeaderDefect): + msg = self._str_msg(string) + self.assertEqual(len(self.get_defects(msg)), 1) + self.assertDefectsEqual( + self.get_defects(msg), [errors.InvalidHeaderDefect] + ) + + if msg: + self.assertEqual(msg.items(), [("Subject", "Dummy subject")]) + self.assertEqual(msg.get_payload(), "body\r\n") + + def test_misplaced_envelope(self): + string = ( + "Subject: Dummy subject\r\nFrom wtf\r\nTo: abc\r\n\r\nbody\r\n" + ) + with self._raise_point(errors.MisplacedEnvelopeHeaderDefect): + msg = self._str_msg(string) + self.assertEqual(len(self.get_defects(msg)), 1) + self.assertDefectsEqual( + self.get_defects(msg), [errors.MisplacedEnvelopeHeaderDefect] + ) + + if msg: + headers = [("Subject", "Dummy subject"), ("To", "abc")] + self.assertEqual(msg.items(), headers) + self.assertEqual(msg.get_payload(), "body\r\n") + + + +class TestCompat32(TestDefectsBase, TestEmailBase): + + policy = policy.compat32 + + def get_defects(self, obj): + return obj.defects + class TestDefectDetection(TestDefectsBase, TestEmailBase): @@ -332,6 +371,9 @@ def _raise_point(self, defect): with self.assertRaises(defect): yield + def get_defects(self, obj): + return obj.defects + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py index 4020f1041c4304..4e6c213510c74c 100644 --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -2263,70 +2263,6 @@ def test_parse_missing_minor_type(self): eq(msg.get_content_maintype(), 'text') eq(msg.get_content_subtype(), 'plain') - # test_defect_handling - def test_same_boundary_inner_outer(self): - msg = self._msgobj('msg_15.txt') - # XXX We can probably eventually do better - inner = msg.get_payload(0) - self.assertHasAttr(inner, 'defects') - self.assertEqual(len(inner.defects), 1) - self.assertIsInstance(inner.defects[0], - errors.StartBoundaryNotFoundDefect) - - # test_defect_handling - def test_multipart_no_boundary(self): - msg = self._msgobj('msg_25.txt') - self.assertIsInstance(msg.get_payload(), str) - self.assertEqual(len(msg.defects), 2) - self.assertIsInstance(msg.defects[0], - errors.NoBoundaryInMultipartDefect) - self.assertIsInstance(msg.defects[1], - errors.MultipartInvariantViolationDefect) - - multipart_msg = textwrap.dedent("""\ - Date: Wed, 14 Nov 2007 12:56:23 GMT - From: foo@bar.invalid - To: foo@bar.invalid - Subject: Content-Transfer-Encoding: base64 and multipart - MIME-Version: 1.0 - Content-Type: multipart/mixed; - boundary="===============3344438784458119861=="{} - - --===============3344438784458119861== - Content-Type: text/plain - - Test message - - --===============3344438784458119861== - Content-Type: application/octet-stream - Content-Transfer-Encoding: base64 - - YWJj - - --===============3344438784458119861==-- - """) - - # test_defect_handling - def test_multipart_invalid_cte(self): - msg = self._str_msg( - self.multipart_msg.format("\nContent-Transfer-Encoding: base64")) - self.assertEqual(len(msg.defects), 1) - self.assertIsInstance(msg.defects[0], - errors.InvalidMultipartContentTransferEncodingDefect) - - # test_defect_handling - def test_multipart_no_cte_no_defect(self): - msg = self._str_msg(self.multipart_msg.format('')) - self.assertEqual(len(msg.defects), 0) - - # test_defect_handling - def test_multipart_valid_cte_no_defect(self): - for cte in ('7bit', '8bit', 'BINary'): - msg = self._str_msg( - self.multipart_msg.format( - "\nContent-Transfer-Encoding: {}".format(cte))) - self.assertEqual(len(msg.defects), 0) - # test_headerregistry.TestContentTypeHeader invalid_1 and invalid_2. def test_invalid_content_type(self): eq = self.assertEqual @@ -2403,30 +2339,6 @@ def test_missing_start_boundary(self): self.assertIsInstance(bad.defects[0], errors.StartBoundaryNotFoundDefect) - # test_defect_handling - def test_first_line_is_continuation_header(self): - eq = self.assertEqual - m = ' Line 1\nSubject: test\n\nbody' - msg = email.message_from_string(m) - eq(msg.keys(), ['Subject']) - eq(msg.get_payload(), 'body') - eq(len(msg.defects), 1) - self.assertDefectsEqual(msg.defects, - [errors.FirstHeaderLineIsContinuationDefect]) - eq(msg.defects[0].line, ' Line 1\n') - - # test_defect_handling - def test_missing_header_body_separator(self): - # Our heuristic if we see a line that doesn't look like a header (no - # leading whitespace but no ':') is to assume that the blank line that - # separates the header from the body is missing, and to stop parsing - # headers and start parsing the body. - msg = self._str_msg('Subject: test\nnot a header\nTo: abc\n\nb\n') - self.assertEqual(msg.keys(), ['Subject']) - self.assertEqual(msg.get_payload(), 'not a header\nTo: abc\n\nb\n') - self.assertDefectsEqual(msg.defects, - [errors.MissingHeaderBodySeparatorDefect]) - def test_string_payload_with_extra_space_after_cte(self): # https://github.com/python/cpython/issues/98188 cte = "base64 " diff --git a/Misc/NEWS.d/next/Library/2025-12-04-09-22-31.gh-issue-68552.I_v-xB.rst b/Misc/NEWS.d/next/Library/2025-12-04-09-22-31.gh-issue-68552.I_v-xB.rst new file mode 100644 index 00000000000000..bd3e53c9f8193f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-04-09-22-31.gh-issue-68552.I_v-xB.rst @@ -0,0 +1 @@ +``MisplacedEnvelopeHeaderDefect`` and ``Missing header name`` defects are now correctly passed to the ``handle_defect`` method of ``policy`` in :class:`~email.parser.FeedParser`. From 332da6295f365b09cabf30a9222323b056ab1410 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Sat, 6 Dec 2025 14:09:10 -0800 Subject: [PATCH 286/638] GH-142363: Contrast and gradient CSS fixes for Tachyon flamegraph (#142364) --- Lib/profiling/sampling/_shared_assets/base.css | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Lib/profiling/sampling/_shared_assets/base.css b/Lib/profiling/sampling/_shared_assets/base.css index 20516913496cbe..d9223a98c0f756 100644 --- a/Lib/profiling/sampling/_shared_assets/base.css +++ b/Lib/profiling/sampling/_shared_assets/base.css @@ -57,9 +57,9 @@ --header-gradient: linear-gradient(135deg, #3776ab 0%, #4584bb 100%); /* Light mode heat palette - blue to yellow to orange to red (cold to hot) */ - --heat-1: #d6e9f8; + --heat-1: #7ba3d1; --heat-2: #a8d0ef; - --heat-3: #7ba3d1; + --heat-3: #d6e9f8; --heat-4: #ffe6a8; --heat-5: #ffd43b; --heat-6: #ffb84d; @@ -104,11 +104,11 @@ --header-gradient: linear-gradient(135deg, #21262d 0%, #30363d 100%); /* Dark mode heat palette - dark blue to teal to yellow to orange (cold to hot) */ - --heat-1: #1e3a5f; - --heat-2: #2d5580; - --heat-3: #4a7ba7; - --heat-4: #5a9fa8; - --heat-5: #7ec488; + --heat-1: #4a7ba7; + --heat-2: #5a9fa8; + --heat-3: #6ab5b5; + --heat-4: #7ec488; + --heat-5: #a0d878; --heat-6: #c4de6a; --heat-7: #f4d44d; --heat-8: #ff6b35; From 572c780aa875e4eb00961743f216214ce6d03f9f Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sat, 6 Dec 2025 22:37:34 +0000 Subject: [PATCH 287/638] gh-138122: Implement frame caching in RemoteUnwinder to reduce memory reads (#142137) This PR implements frame caching in the RemoteUnwinder class to significantly reduce memory reads when profiling remote processes with deep call stacks. When cache_frames=True, the unwinder stores the frame chain from each sample and reuses unchanged portions in subsequent samples. Since most profiling samples capture similar call stacks (especially the parent frames), this optimization avoids repeatedly reading the same frame data from the target process. The implementation adds a last_profiled_frame field to the thread state that tracks where the previous sample stopped. On the next sample, if the current frame chain reaches this marker, the cached frames from that point onward are reused instead of being re-read from remote memory. The sampling profiler now enables frame caching by default. --- Include/cpython/pystate.h | 2 + Include/internal/pycore_debug_offsets.h | 2 + .../pycore_global_objects_fini_generated.h | 2 + Include/internal/pycore_global_strings.h | 2 + .../internal/pycore_runtime_init_generated.h | 2 + .../internal/pycore_unicodeobject_generated.h | 8 + InternalDocs/frames.md | 20 + Lib/profiling/sampling/sample.py | 112 ++- Lib/test/test_external_inspection.py | 762 ++++++++++++++++++ ...-12-01-14-43-58.gh-issue-138122.nRm3ic.rst | 5 + Modules/Setup.stdlib.in | 2 +- Modules/_remote_debugging/_remote_debugging.h | 76 +- Modules/_remote_debugging/clinic/module.c.h | 92 ++- Modules/_remote_debugging/code_objects.c | 5 + Modules/_remote_debugging/frame_cache.c | 236 ++++++ Modules/_remote_debugging/frames.c | 263 +++++- Modules/_remote_debugging/module.c | 146 +++- Modules/_remote_debugging/threads.c | 38 +- PCbuild/_remote_debugging.vcxproj | 1 + PCbuild/_remote_debugging.vcxproj.filters | 3 + Python/ceval.c | 10 + Python/remote_debug.h | 109 +++ Python/remote_debugging.c | 97 +-- .../benchmark_external_inspection.py | 2 +- 24 files changed, 1855 insertions(+), 142 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-12-01-14-43-58.gh-issue-138122.nRm3ic.rst create mode 100644 Modules/_remote_debugging/frame_cache.c diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 1e1e46ea4c0bcd..08d71070ddccd3 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -135,6 +135,8 @@ struct _ts { /* Pointer to currently executing frame. */ struct _PyInterpreterFrame *current_frame; + struct _PyInterpreterFrame *last_profiled_frame; + Py_tracefunc c_profilefunc; Py_tracefunc c_tracefunc; PyObject *c_profileobj; diff --git a/Include/internal/pycore_debug_offsets.h b/Include/internal/pycore_debug_offsets.h index 0f17bf17f82656..bfd86c08887b08 100644 --- a/Include/internal/pycore_debug_offsets.h +++ b/Include/internal/pycore_debug_offsets.h @@ -102,6 +102,7 @@ typedef struct _Py_DebugOffsets { uint64_t next; uint64_t interp; uint64_t current_frame; + uint64_t last_profiled_frame; uint64_t thread_id; uint64_t native_thread_id; uint64_t datastack_chunk; @@ -272,6 +273,7 @@ typedef struct _Py_DebugOffsets { .next = offsetof(PyThreadState, next), \ .interp = offsetof(PyThreadState, interp), \ .current_frame = offsetof(PyThreadState, current_frame), \ + .last_profiled_frame = offsetof(PyThreadState, last_profiled_frame), \ .thread_id = offsetof(PyThreadState, thread_id), \ .native_thread_id = offsetof(PyThreadState, native_thread_id), \ .datastack_chunk = offsetof(PyThreadState, datastack_chunk), \ diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 783747d1f01580..d23d6d4f91bc28 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1609,6 +1609,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_parameter_type)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_return)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(c_stack)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cache_frames)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cached_datetime_module)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cached_statements)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cadata)); @@ -2053,6 +2054,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(stacklevel)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(start)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(statement)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(stats)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(status)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(stderr)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(stdin)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 374617d8284b48..5c3ea474ad09b7 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -332,6 +332,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(c_parameter_type) STRUCT_FOR_ID(c_return) STRUCT_FOR_ID(c_stack) + STRUCT_FOR_ID(cache_frames) STRUCT_FOR_ID(cached_datetime_module) STRUCT_FOR_ID(cached_statements) STRUCT_FOR_ID(cadata) @@ -776,6 +777,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(stacklevel) STRUCT_FOR_ID(start) STRUCT_FOR_ID(statement) + STRUCT_FOR_ID(stats) STRUCT_FOR_ID(status) STRUCT_FOR_ID(stderr) STRUCT_FOR_ID(stdin) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index a66c97f7f13677..31d88339a13425 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1607,6 +1607,7 @@ extern "C" { INIT_ID(c_parameter_type), \ INIT_ID(c_return), \ INIT_ID(c_stack), \ + INIT_ID(cache_frames), \ INIT_ID(cached_datetime_module), \ INIT_ID(cached_statements), \ INIT_ID(cadata), \ @@ -2051,6 +2052,7 @@ extern "C" { INIT_ID(stacklevel), \ INIT_ID(start), \ INIT_ID(statement), \ + INIT_ID(stats), \ INIT_ID(status), \ INIT_ID(stderr), \ INIT_ID(stdin), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 2061b1d204951d..c5b01ff9876643 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1108,6 +1108,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(cache_frames); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(cached_datetime_module); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2884,6 +2888,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(stats); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(status); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/InternalDocs/frames.md b/InternalDocs/frames.md index 804d7436018a10..d56e5481d3cbfe 100644 --- a/InternalDocs/frames.md +++ b/InternalDocs/frames.md @@ -111,6 +111,26 @@ The shim frame points to a special code object containing the `INTERPRETER_EXIT` instruction which cleans up the shim frame and returns. +### Remote Profiling Frame Cache + +The `last_profiled_frame` field in `PyThreadState` supports an optimization for +remote profilers that sample call stacks from external processes. When a remote +profiler reads the call stack, it writes the current frame address to this field. +The eval loop then keeps this pointer valid by updating it to the parent frame +whenever a frame returns (in `_PyEval_FrameClearAndPop`). + +This creates a "high-water mark" that always points to a frame still on the stack. +On subsequent samples, the profiler can walk from `current_frame` until it reaches +`last_profiled_frame`, knowing that frames from that point downward are unchanged +and can be retrieved from a cache. This significantly reduces the amount of remote +memory reads needed when call stacks are deep and stable at their base. + +The update in `_PyEval_FrameClearAndPop` is guarded: it only writes when +`last_profiled_frame` is non-NULL AND matches the frame being popped. This +prevents transient frames (called and returned between profiler samples) from +corrupting the cache pointer, while avoiding any overhead when profiling is inactive. + + ### The Instruction Pointer `_PyInterpreterFrame` has two fields which are used to maintain the instruction diff --git a/Lib/profiling/sampling/sample.py b/Lib/profiling/sampling/sample.py index 99cac71a4049a6..dd4ea1edbf668d 100644 --- a/Lib/profiling/sampling/sample.py +++ b/Lib/profiling/sampling/sample.py @@ -27,21 +27,24 @@ class SampleProfiler: - def __init__(self, pid, sample_interval_usec, all_threads, *, mode=PROFILING_MODE_WALL, native=False, gc=True, skip_non_matching_threads=True): + def __init__(self, pid, sample_interval_usec, all_threads, *, mode=PROFILING_MODE_WALL, native=False, gc=True, skip_non_matching_threads=True, collect_stats=False): self.pid = pid self.sample_interval_usec = sample_interval_usec self.all_threads = all_threads self.mode = mode # Store mode for later use + self.collect_stats = collect_stats if _FREE_THREADED_BUILD: self.unwinder = _remote_debugging.RemoteUnwinder( self.pid, all_threads=self.all_threads, mode=mode, native=native, gc=gc, - skip_non_matching_threads=skip_non_matching_threads + skip_non_matching_threads=skip_non_matching_threads, cache_frames=True, + stats=collect_stats ) else: only_active_threads = bool(self.all_threads) self.unwinder = _remote_debugging.RemoteUnwinder( self.pid, only_active_thread=only_active_threads, mode=mode, native=native, gc=gc, - skip_non_matching_threads=skip_non_matching_threads + skip_non_matching_threads=skip_non_matching_threads, cache_frames=True, + stats=collect_stats ) # Track sample intervals and total sample count self.sample_intervals = deque(maxlen=100) @@ -129,6 +132,10 @@ def sample(self, collector, duration_sec=10, *, async_aware=False): print(f"Sample rate: {sample_rate:.2f} samples/sec") print(f"Error rate: {error_rate:.2f}%") + # Print unwinder stats if stats collection is enabled + if self.collect_stats: + self._print_unwinder_stats() + # Pass stats to flamegraph collector if it's the right type if hasattr(collector, 'set_stats'): collector.set_stats(self.sample_interval_usec, running_time, sample_rate, error_rate, missed_samples, mode=self.mode) @@ -176,17 +183,100 @@ def _print_realtime_stats(self): (1.0 / min_hz) * 1_000_000 if min_hz > 0 else 0 ) # Max time = Min Hz + # Build cache stats string if stats collection is enabled + cache_stats_str = "" + if self.collect_stats: + try: + stats = self.unwinder.get_stats() + hits = stats.get('frame_cache_hits', 0) + partial = stats.get('frame_cache_partial_hits', 0) + misses = stats.get('frame_cache_misses', 0) + total = hits + partial + misses + if total > 0: + hit_pct = (hits + partial) / total * 100 + cache_stats_str = f" {ANSIColors.MAGENTA}Cache: {hit_pct:.1f}% ({hits}+{partial}/{misses}){ANSIColors.RESET}" + except RuntimeError: + pass + # Clear line and print stats print( - f"\r\033[K{ANSIColors.BOLD_BLUE}Real-time sampling stats:{ANSIColors.RESET} " - f"{ANSIColors.YELLOW}Mean: {mean_hz:.1f}Hz ({mean_us_per_sample:.2f}µs){ANSIColors.RESET} " - f"{ANSIColors.GREEN}Min: {min_hz:.1f}Hz ({max_us_per_sample:.2f}µs){ANSIColors.RESET} " - f"{ANSIColors.RED}Max: {max_hz:.1f}Hz ({min_us_per_sample:.2f}µs){ANSIColors.RESET} " - f"{ANSIColors.CYAN}Samples: {self.total_samples}{ANSIColors.RESET}", + f"\r\033[K{ANSIColors.BOLD_BLUE}Stats:{ANSIColors.RESET} " + f"{ANSIColors.YELLOW}{mean_hz:.1f}Hz ({mean_us_per_sample:.1f}µs){ANSIColors.RESET} " + f"{ANSIColors.GREEN}Min: {min_hz:.1f}Hz{ANSIColors.RESET} " + f"{ANSIColors.RED}Max: {max_hz:.1f}Hz{ANSIColors.RESET} " + f"{ANSIColors.CYAN}N={self.total_samples}{ANSIColors.RESET}" + f"{cache_stats_str}", end="", flush=True, ) + def _print_unwinder_stats(self): + """Print unwinder statistics including cache performance.""" + try: + stats = self.unwinder.get_stats() + except RuntimeError: + return # Stats not enabled + + print(f"\n{ANSIColors.BOLD_BLUE}{'='*50}{ANSIColors.RESET}") + print(f"{ANSIColors.BOLD_BLUE}Unwinder Statistics:{ANSIColors.RESET}") + + # Frame cache stats + total_samples = stats.get('total_samples', 0) + frame_cache_hits = stats.get('frame_cache_hits', 0) + frame_cache_partial_hits = stats.get('frame_cache_partial_hits', 0) + frame_cache_misses = stats.get('frame_cache_misses', 0) + total_lookups = frame_cache_hits + frame_cache_partial_hits + frame_cache_misses + + # Calculate percentages + hits_pct = (frame_cache_hits / total_lookups * 100) if total_lookups > 0 else 0 + partial_pct = (frame_cache_partial_hits / total_lookups * 100) if total_lookups > 0 else 0 + misses_pct = (frame_cache_misses / total_lookups * 100) if total_lookups > 0 else 0 + + print(f" {ANSIColors.CYAN}Frame Cache:{ANSIColors.RESET}") + print(f" Total samples: {total_samples:,}") + print(f" Full hits: {frame_cache_hits:,} ({ANSIColors.GREEN}{hits_pct:.1f}%{ANSIColors.RESET})") + print(f" Partial hits: {frame_cache_partial_hits:,} ({ANSIColors.YELLOW}{partial_pct:.1f}%{ANSIColors.RESET})") + print(f" Misses: {frame_cache_misses:,} ({ANSIColors.RED}{misses_pct:.1f}%{ANSIColors.RESET})") + + # Frame read stats + frames_from_cache = stats.get('frames_read_from_cache', 0) + frames_from_memory = stats.get('frames_read_from_memory', 0) + total_frames = frames_from_cache + frames_from_memory + cache_frame_pct = (frames_from_cache / total_frames * 100) if total_frames > 0 else 0 + memory_frame_pct = (frames_from_memory / total_frames * 100) if total_frames > 0 else 0 + + print(f" {ANSIColors.CYAN}Frame Reads:{ANSIColors.RESET}") + print(f" From cache: {frames_from_cache:,} ({ANSIColors.GREEN}{cache_frame_pct:.1f}%{ANSIColors.RESET})") + print(f" From memory: {frames_from_memory:,} ({ANSIColors.RED}{memory_frame_pct:.1f}%{ANSIColors.RESET})") + + # Code object cache stats + code_hits = stats.get('code_object_cache_hits', 0) + code_misses = stats.get('code_object_cache_misses', 0) + total_code = code_hits + code_misses + code_hits_pct = (code_hits / total_code * 100) if total_code > 0 else 0 + code_misses_pct = (code_misses / total_code * 100) if total_code > 0 else 0 + + print(f" {ANSIColors.CYAN}Code Object Cache:{ANSIColors.RESET}") + print(f" Hits: {code_hits:,} ({ANSIColors.GREEN}{code_hits_pct:.1f}%{ANSIColors.RESET})") + print(f" Misses: {code_misses:,} ({ANSIColors.RED}{code_misses_pct:.1f}%{ANSIColors.RESET})") + + # Memory operations + memory_reads = stats.get('memory_reads', 0) + memory_bytes = stats.get('memory_bytes_read', 0) + if memory_bytes >= 1024 * 1024: + memory_str = f"{memory_bytes / (1024 * 1024):.1f} MB" + elif memory_bytes >= 1024: + memory_str = f"{memory_bytes / 1024:.1f} KB" + else: + memory_str = f"{memory_bytes} B" + print(f" {ANSIColors.CYAN}Memory:{ANSIColors.RESET}") + print(f" Read operations: {memory_reads:,} ({memory_str})") + + # Stale invalidations + stale_invalidations = stats.get('stale_cache_invalidations', 0) + if stale_invalidations > 0: + print(f" {ANSIColors.YELLOW}Stale cache invalidations: {stale_invalidations}{ANSIColors.RESET}") + def sample( pid, @@ -234,7 +324,8 @@ def sample( mode=mode, native=native, gc=gc, - skip_non_matching_threads=skip_non_matching_threads + skip_non_matching_threads=skip_non_matching_threads, + collect_stats=realtime_stats, ) profiler.realtime_stats = realtime_stats @@ -290,7 +381,8 @@ def sample_live( mode=mode, native=native, gc=gc, - skip_non_matching_threads=skip_non_matching_threads + skip_non_matching_threads=skip_non_matching_threads, + collect_stats=realtime_stats, ) profiler.realtime_stats = realtime_stats diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 7decd8f32d5a2b..2e6e6eaad06450 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -1,3 +1,4 @@ +import contextlib import unittest import os import textwrap @@ -2038,5 +2039,766 @@ def busy_thread(): p.stderr.close() +class TestFrameCaching(unittest.TestCase): + """Test that frame caching produces correct results. + + Uses socket-based synchronization for deterministic testing. + All tests verify cache reuse via object identity checks (assertIs). + """ + + maxDiff = None + MAX_TRIES = 10 + + @contextlib.contextmanager + def _target_process(self, script_body): + """Context manager for running a target process with socket sync.""" + port = find_unused_port() + script = f"""\ +import socket +sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +sock.connect(('localhost', {port})) +{textwrap.dedent(script_body)} +""" + + with os_helper.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + + server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server_socket.bind(("localhost", port)) + server_socket.settimeout(SHORT_TIMEOUT) + server_socket.listen(1) + + script_name = _make_test_script(script_dir, "script", script) + client_socket = None + p = None + try: + p = subprocess.Popen([sys.executable, script_name]) + client_socket, _ = server_socket.accept() + server_socket.close() + + def make_unwinder(cache_frames=True): + return RemoteUnwinder(p.pid, all_threads=True, cache_frames=cache_frames) + + yield p, client_socket, make_unwinder + + except PermissionError: + self.skipTest("Insufficient permissions to read the stack trace") + finally: + if client_socket: + client_socket.close() + if p: + p.kill() + p.terminate() + p.wait(timeout=SHORT_TIMEOUT) + + def _wait_for_signal(self, client_socket, signal): + """Block until signal received from target.""" + response = b"" + while signal not in response: + chunk = client_socket.recv(64) + if not chunk: + break + response += chunk + return response + + def _get_frames(self, unwinder, required_funcs): + """Sample and return frame_info list for thread containing required_funcs.""" + traces = unwinder.get_stack_trace() + for interp in traces: + for thread in interp.threads: + funcs = [f.funcname for f in thread.frame_info] + if required_funcs.issubset(set(funcs)): + return thread.frame_info + return None + + def _sample_frames(self, client_socket, unwinder, wait_signal, send_ack, required_funcs, expected_frames=1): + """Wait for signal, sample frames, send ack. Returns frame_info list.""" + self._wait_for_signal(client_socket, wait_signal) + # Give at least MAX_TRIES tries for the process to arrive to a steady state + for _ in range(self.MAX_TRIES): + frames = self._get_frames(unwinder, required_funcs) + if frames and len(frames) >= expected_frames: + break + time.sleep(0.1) + client_socket.sendall(send_ack) + return frames + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_cache_hit_same_stack(self): + """Test that consecutive samples reuse cached parent frame objects. + + The current frame (index 0) is always re-read from memory to get + updated line numbers, so it may be a different object. Parent frames + (index 1+) should be identical objects from cache. + """ + script_body = """\ + def level3(): + sock.sendall(b"sync1") + sock.recv(16) + sock.sendall(b"sync2") + sock.recv(16) + sock.sendall(b"sync3") + sock.recv(16) + + def level2(): + level3() + + def level1(): + level2() + + level1() + """ + + with self._target_process(script_body) as (p, client_socket, make_unwinder): + unwinder = make_unwinder(cache_frames=True) + expected = {"level1", "level2", "level3"} + + frames1 = self._sample_frames(client_socket, unwinder, b"sync1", b"ack", expected) + frames2 = self._sample_frames(client_socket, unwinder, b"sync2", b"ack", expected) + frames3 = self._sample_frames(client_socket, unwinder, b"sync3", b"done", expected) + + self.assertIsNotNone(frames1) + self.assertIsNotNone(frames2) + self.assertIsNotNone(frames3) + self.assertEqual(len(frames1), len(frames2)) + self.assertEqual(len(frames2), len(frames3)) + + # Current frame (index 0) is always re-read, so check value equality + self.assertEqual(frames1[0].funcname, frames2[0].funcname) + self.assertEqual(frames2[0].funcname, frames3[0].funcname) + + # Parent frames (index 1+) must be identical objects (cache reuse) + for i in range(1, len(frames1)): + f1, f2, f3 = frames1[i], frames2[i], frames3[i] + self.assertIs(f1, f2, f"Frame {i}: samples 1-2 must be same object") + self.assertIs(f2, f3, f"Frame {i}: samples 2-3 must be same object") + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_line_number_updates_in_same_frame(self): + """Test that line numbers are correctly updated when execution moves within a function. + + When the profiler samples at different points within the same function, + it must report the correct line number for each sample, not stale cached values. + """ + script_body = """\ + def outer(): + inner() + + def inner(): + sock.sendall(b"line_a"); sock.recv(16) + sock.sendall(b"line_b"); sock.recv(16) + sock.sendall(b"line_c"); sock.recv(16) + sock.sendall(b"line_d"); sock.recv(16) + + outer() + """ + + with self._target_process(script_body) as (p, client_socket, make_unwinder): + unwinder = make_unwinder(cache_frames=True) + + frames_a = self._sample_frames(client_socket, unwinder, b"line_a", b"ack", {"inner"}) + frames_b = self._sample_frames(client_socket, unwinder, b"line_b", b"ack", {"inner"}) + frames_c = self._sample_frames(client_socket, unwinder, b"line_c", b"ack", {"inner"}) + frames_d = self._sample_frames(client_socket, unwinder, b"line_d", b"done", {"inner"}) + + self.assertIsNotNone(frames_a) + self.assertIsNotNone(frames_b) + self.assertIsNotNone(frames_c) + self.assertIsNotNone(frames_d) + + # Get the 'inner' frame from each sample (should be index 0) + inner_a = frames_a[0] + inner_b = frames_b[0] + inner_c = frames_c[0] + inner_d = frames_d[0] + + self.assertEqual(inner_a.funcname, "inner") + self.assertEqual(inner_b.funcname, "inner") + self.assertEqual(inner_c.funcname, "inner") + self.assertEqual(inner_d.funcname, "inner") + + # Line numbers must be different and increasing (execution moves forward) + self.assertLess(inner_a.lineno, inner_b.lineno, + "Line B should be after line A") + self.assertLess(inner_b.lineno, inner_c.lineno, + "Line C should be after line B") + self.assertLess(inner_c.lineno, inner_d.lineno, + "Line D should be after line C") + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_cache_invalidation_on_return(self): + """Test cache invalidation when stack shrinks (function returns).""" + script_body = """\ + def inner(): + sock.sendall(b"at_inner") + sock.recv(16) + + def outer(): + inner() + sock.sendall(b"at_outer") + sock.recv(16) + + outer() + """ + + with self._target_process(script_body) as (p, client_socket, make_unwinder): + unwinder = make_unwinder(cache_frames=True) + + frames_deep = self._sample_frames( + client_socket, unwinder, b"at_inner", b"ack", {"inner", "outer"}) + frames_shallow = self._sample_frames( + client_socket, unwinder, b"at_outer", b"done", {"outer"}) + + self.assertIsNotNone(frames_deep) + self.assertIsNotNone(frames_shallow) + + funcs_deep = [f.funcname for f in frames_deep] + funcs_shallow = [f.funcname for f in frames_shallow] + + self.assertIn("inner", funcs_deep) + self.assertIn("outer", funcs_deep) + self.assertNotIn("inner", funcs_shallow) + self.assertIn("outer", funcs_shallow) + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_cache_invalidation_on_call(self): + """Test cache invalidation when stack grows (new function called).""" + script_body = """\ + def deeper(): + sock.sendall(b"at_deeper") + sock.recv(16) + + def middle(): + sock.sendall(b"at_middle") + sock.recv(16) + deeper() + + def top(): + middle() + + top() + """ + + with self._target_process(script_body) as (p, client_socket, make_unwinder): + unwinder = make_unwinder(cache_frames=True) + + frames_before = self._sample_frames( + client_socket, unwinder, b"at_middle", b"ack", {"middle", "top"}) + frames_after = self._sample_frames( + client_socket, unwinder, b"at_deeper", b"done", {"deeper", "middle", "top"}) + + self.assertIsNotNone(frames_before) + self.assertIsNotNone(frames_after) + + funcs_before = [f.funcname for f in frames_before] + funcs_after = [f.funcname for f in frames_after] + + self.assertIn("middle", funcs_before) + self.assertIn("top", funcs_before) + self.assertNotIn("deeper", funcs_before) + + self.assertIn("deeper", funcs_after) + self.assertIn("middle", funcs_after) + self.assertIn("top", funcs_after) + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_partial_stack_reuse(self): + """Test that unchanged bottom frames are reused when top changes (A→B→C to A→B→D).""" + script_body = """\ + def func_c(): + sock.sendall(b"at_c") + sock.recv(16) + + def func_d(): + sock.sendall(b"at_d") + sock.recv(16) + + def func_b(): + func_c() + func_d() + + def func_a(): + func_b() + + func_a() + """ + + with self._target_process(script_body) as (p, client_socket, make_unwinder): + unwinder = make_unwinder(cache_frames=True) + + # Sample at C: stack is A→B→C + frames_c = self._sample_frames( + client_socket, unwinder, b"at_c", b"ack", {"func_a", "func_b", "func_c"}) + # Sample at D: stack is A→B→D (C returned, D called) + frames_d = self._sample_frames( + client_socket, unwinder, b"at_d", b"done", {"func_a", "func_b", "func_d"}) + + self.assertIsNotNone(frames_c) + self.assertIsNotNone(frames_d) + + # Find func_a and func_b frames in both samples + def find_frame(frames, funcname): + for f in frames: + if f.funcname == funcname: + return f + return None + + frame_a_in_c = find_frame(frames_c, "func_a") + frame_b_in_c = find_frame(frames_c, "func_b") + frame_a_in_d = find_frame(frames_d, "func_a") + frame_b_in_d = find_frame(frames_d, "func_b") + + self.assertIsNotNone(frame_a_in_c) + self.assertIsNotNone(frame_b_in_c) + self.assertIsNotNone(frame_a_in_d) + self.assertIsNotNone(frame_b_in_d) + + # The bottom frames (A, B) should be the SAME objects (cache reuse) + self.assertIs(frame_a_in_c, frame_a_in_d, "func_a frame should be reused from cache") + self.assertIs(frame_b_in_c, frame_b_in_d, "func_b frame should be reused from cache") + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_recursive_frames(self): + """Test caching with same function appearing multiple times (recursion).""" + script_body = """\ + def recurse(n): + if n <= 0: + sock.sendall(b"sync1") + sock.recv(16) + sock.sendall(b"sync2") + sock.recv(16) + else: + recurse(n - 1) + + recurse(5) + """ + + with self._target_process(script_body) as (p, client_socket, make_unwinder): + unwinder = make_unwinder(cache_frames=True) + + frames1 = self._sample_frames( + client_socket, unwinder, b"sync1", b"ack", {"recurse"}) + frames2 = self._sample_frames( + client_socket, unwinder, b"sync2", b"done", {"recurse"}) + + self.assertIsNotNone(frames1) + self.assertIsNotNone(frames2) + + # Should have multiple "recurse" frames (6 total: recurse(5) down to recurse(0)) + recurse_count = sum(1 for f in frames1 if f.funcname == "recurse") + self.assertEqual(recurse_count, 6, "Should have 6 recursive frames") + + self.assertEqual(len(frames1), len(frames2)) + + # Current frame (index 0) is re-read, check value equality + self.assertEqual(frames1[0].funcname, frames2[0].funcname) + + # Parent frames (index 1+) should be identical objects (cache reuse) + for i in range(1, len(frames1)): + self.assertIs(frames1[i], frames2[i], + f"Frame {i}: recursive frames must be same object") + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_cache_vs_no_cache_equivalence(self): + """Test that cache_frames=True and cache_frames=False produce equivalent results.""" + script_body = """\ + def level3(): + sock.sendall(b"ready"); sock.recv(16) + + def level2(): + level3() + + def level1(): + level2() + + level1() + """ + + with self._target_process(script_body) as (p, client_socket, make_unwinder): + self._wait_for_signal(client_socket, b"ready") + + # Sample with cache + unwinder_cache = make_unwinder(cache_frames=True) + frames_cached = self._get_frames(unwinder_cache, {"level1", "level2", "level3"}) + + # Sample without cache + unwinder_no_cache = make_unwinder(cache_frames=False) + frames_no_cache = self._get_frames(unwinder_no_cache, {"level1", "level2", "level3"}) + + client_socket.sendall(b"done") + + self.assertIsNotNone(frames_cached) + self.assertIsNotNone(frames_no_cache) + + # Same number of frames + self.assertEqual(len(frames_cached), len(frames_no_cache)) + + # Same function names in same order + funcs_cached = [f.funcname for f in frames_cached] + funcs_no_cache = [f.funcname for f in frames_no_cache] + self.assertEqual(funcs_cached, funcs_no_cache) + + # Same line numbers + lines_cached = [f.lineno for f in frames_cached] + lines_no_cache = [f.lineno for f in frames_no_cache] + self.assertEqual(lines_cached, lines_no_cache) + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_cache_per_thread_isolation(self): + """Test that frame cache is per-thread and cache invalidation works independently.""" + script_body = """\ + import threading + + lock = threading.Lock() + + def sync(msg): + with lock: + sock.sendall(msg + b"\\n") + sock.recv(1) + + # Thread 1 functions + def baz1(): + sync(b"t1:baz1") + + def bar1(): + baz1() + + def blech1(): + sync(b"t1:blech1") + + def foo1(): + bar1() # Goes down to baz1, syncs + blech1() # Returns up, goes down to blech1, syncs + + # Thread 2 functions + def baz2(): + sync(b"t2:baz2") + + def bar2(): + baz2() + + def blech2(): + sync(b"t2:blech2") + + def foo2(): + bar2() # Goes down to baz2, syncs + blech2() # Returns up, goes down to blech2, syncs + + t1 = threading.Thread(target=foo1) + t2 = threading.Thread(target=foo2) + t1.start() + t2.start() + t1.join() + t2.join() + """ + + with self._target_process(script_body) as (p, client_socket, make_unwinder): + unwinder = make_unwinder(cache_frames=True) + buffer = b"" + + def recv_msg(): + """Receive a single message from socket.""" + nonlocal buffer + while b"\n" not in buffer: + chunk = client_socket.recv(256) + if not chunk: + return None + buffer += chunk + msg, buffer = buffer.split(b"\n", 1) + return msg + + def get_thread_frames(target_funcs): + """Get frames for thread matching target functions.""" + retries = 0 + for _ in busy_retry(SHORT_TIMEOUT): + if retries >= 5: + break + retries += 1 + # On Windows, ReadProcessMemory can fail with OSError + # (WinError 299) when frame pointers are in flux + with contextlib.suppress(RuntimeError, OSError): + traces = unwinder.get_stack_trace() + for interp in traces: + for thread in interp.threads: + funcs = [f.funcname for f in thread.frame_info] + if any(f in funcs for f in target_funcs): + return funcs + return None + + # Track results for each sync point + results = {} + + # Process 4 sync points: baz1, baz2, blech1, blech2 + # With the lock, threads are serialized - handle one at a time + for _ in range(4): + msg = recv_msg() + self.assertIsNotNone(msg, "Expected message from subprocess") + + # Determine which thread/function and take snapshot + if msg == b"t1:baz1": + funcs = get_thread_frames(["baz1", "bar1", "foo1"]) + self.assertIsNotNone(funcs, "Thread 1 not found at baz1") + results["t1:baz1"] = funcs + elif msg == b"t2:baz2": + funcs = get_thread_frames(["baz2", "bar2", "foo2"]) + self.assertIsNotNone(funcs, "Thread 2 not found at baz2") + results["t2:baz2"] = funcs + elif msg == b"t1:blech1": + funcs = get_thread_frames(["blech1", "foo1"]) + self.assertIsNotNone(funcs, "Thread 1 not found at blech1") + results["t1:blech1"] = funcs + elif msg == b"t2:blech2": + funcs = get_thread_frames(["blech2", "foo2"]) + self.assertIsNotNone(funcs, "Thread 2 not found at blech2") + results["t2:blech2"] = funcs + + # Release thread to continue + client_socket.sendall(b"k") + + # Validate Phase 1: baz snapshots + t1_baz = results.get("t1:baz1") + t2_baz = results.get("t2:baz2") + self.assertIsNotNone(t1_baz, "Missing t1:baz1 snapshot") + self.assertIsNotNone(t2_baz, "Missing t2:baz2 snapshot") + + # Thread 1 at baz1: should have foo1->bar1->baz1 + self.assertIn("baz1", t1_baz) + self.assertIn("bar1", t1_baz) + self.assertIn("foo1", t1_baz) + self.assertNotIn("blech1", t1_baz) + # No cross-contamination + self.assertNotIn("baz2", t1_baz) + self.assertNotIn("bar2", t1_baz) + self.assertNotIn("foo2", t1_baz) + + # Thread 2 at baz2: should have foo2->bar2->baz2 + self.assertIn("baz2", t2_baz) + self.assertIn("bar2", t2_baz) + self.assertIn("foo2", t2_baz) + self.assertNotIn("blech2", t2_baz) + # No cross-contamination + self.assertNotIn("baz1", t2_baz) + self.assertNotIn("bar1", t2_baz) + self.assertNotIn("foo1", t2_baz) + + # Validate Phase 2: blech snapshots (cache invalidation test) + t1_blech = results.get("t1:blech1") + t2_blech = results.get("t2:blech2") + self.assertIsNotNone(t1_blech, "Missing t1:blech1 snapshot") + self.assertIsNotNone(t2_blech, "Missing t2:blech2 snapshot") + + # Thread 1 at blech1: bar1/baz1 should be GONE (cache invalidated) + self.assertIn("blech1", t1_blech) + self.assertIn("foo1", t1_blech) + self.assertNotIn("bar1", t1_blech, "Cache not invalidated: bar1 still present") + self.assertNotIn("baz1", t1_blech, "Cache not invalidated: baz1 still present") + # No cross-contamination + self.assertNotIn("blech2", t1_blech) + + # Thread 2 at blech2: bar2/baz2 should be GONE (cache invalidated) + self.assertIn("blech2", t2_blech) + self.assertIn("foo2", t2_blech) + self.assertNotIn("bar2", t2_blech, "Cache not invalidated: bar2 still present") + self.assertNotIn("baz2", t2_blech, "Cache not invalidated: baz2 still present") + # No cross-contamination + self.assertNotIn("blech1", t2_blech) + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_new_unwinder_with_stale_last_profiled_frame(self): + """Test that a new unwinder returns complete stack when cache lookup misses.""" + script_body = """\ + def level4(): + sock.sendall(b"sync1") + sock.recv(16) + sock.sendall(b"sync2") + sock.recv(16) + + def level3(): + level4() + + def level2(): + level3() + + def level1(): + level2() + + level1() + """ + + with self._target_process(script_body) as (p, client_socket, make_unwinder): + expected = {"level1", "level2", "level3", "level4"} + + # First unwinder samples - this sets last_profiled_frame in target + unwinder1 = make_unwinder(cache_frames=True) + frames1 = self._sample_frames(client_socket, unwinder1, b"sync1", b"ack", expected) + + # Create NEW unwinder (empty cache) and sample + # The target still has last_profiled_frame set from unwinder1 + unwinder2 = make_unwinder(cache_frames=True) + frames2 = self._sample_frames(client_socket, unwinder2, b"sync2", b"done", expected) + + self.assertIsNotNone(frames1) + self.assertIsNotNone(frames2) + + funcs1 = [f.funcname for f in frames1] + funcs2 = [f.funcname for f in frames2] + + # Both should have all levels + for level in ["level1", "level2", "level3", "level4"]: + self.assertIn(level, funcs1, f"{level} missing from first sample") + self.assertIn(level, funcs2, f"{level} missing from second sample") + + # Should have same stack depth + self.assertEqual(len(frames1), len(frames2), + "New unwinder should return complete stack despite stale last_profiled_frame") + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_cache_exhaustion(self): + """Test cache works when frame limit (1024) is exceeded. + + FRAME_CACHE_MAX_FRAMES=1024. With 1100 recursive frames, + the cache can't store all of them but should still work. + """ + # Use 1100 to exceed FRAME_CACHE_MAX_FRAMES=1024 + depth = 1100 + script_body = f"""\ +import sys +sys.setrecursionlimit(2000) + +def recurse(n): + if n <= 0: + sock.sendall(b"ready") + sock.recv(16) # wait for ack + sock.sendall(b"ready2") + sock.recv(16) # wait for done + return + recurse(n - 1) + +recurse({depth}) +""" + + with self._target_process(script_body) as (p, client_socket, make_unwinder): + unwinder_cache = make_unwinder(cache_frames=True) + unwinder_no_cache = make_unwinder(cache_frames=False) + + frames_cached = self._sample_frames( + client_socket, unwinder_cache, b"ready", b"ack", {"recurse"}, expected_frames=1102 + ) + # Sample again with no cache for comparison + frames_no_cache = self._sample_frames( + client_socket, unwinder_no_cache, b"ready2", b"done", {"recurse"}, expected_frames=1102 + ) + + self.assertIsNotNone(frames_cached) + self.assertIsNotNone(frames_no_cache) + + # Both should have many recurse frames (> 1024 limit) + cached_count = [f.funcname for f in frames_cached].count("recurse") + no_cache_count = [f.funcname for f in frames_no_cache].count("recurse") + + self.assertGreater(cached_count, 1000, "Should have >1000 recurse frames") + self.assertGreater(no_cache_count, 1000, "Should have >1000 recurse frames") + + # Both modes should produce same frame count + self.assertEqual(len(frames_cached), len(frames_no_cache), + "Cache exhaustion should not affect stack completeness") + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_get_stats(self): + """Test that get_stats() returns statistics when stats=True.""" + script_body = """\ + sock.sendall(b"ready") + sock.recv(16) + """ + + with self._target_process(script_body) as (p, client_socket, _): + unwinder = RemoteUnwinder(p.pid, all_threads=True, stats=True) + self._wait_for_signal(client_socket, b"ready") + + # Take a sample + unwinder.get_stack_trace() + + stats = unwinder.get_stats() + client_socket.sendall(b"done") + + # Verify expected keys exist + expected_keys = [ + 'total_samples', 'frame_cache_hits', 'frame_cache_misses', + 'frame_cache_partial_hits', 'frames_read_from_cache', + 'frames_read_from_memory', 'frame_cache_hit_rate' + ] + for key in expected_keys: + self.assertIn(key, stats) + + self.assertEqual(stats['total_samples'], 1) + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_get_stats_disabled_raises(self): + """Test that get_stats() raises RuntimeError when stats=False.""" + script_body = """\ + sock.sendall(b"ready") + sock.recv(16) + """ + + with self._target_process(script_body) as (p, client_socket, _): + unwinder = RemoteUnwinder(p.pid, all_threads=True) # stats=False by default + self._wait_for_signal(client_socket, b"ready") + + with self.assertRaises(RuntimeError): + unwinder.get_stats() + + client_socket.sendall(b"done") + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2025-12-01-14-43-58.gh-issue-138122.nRm3ic.rst b/Misc/NEWS.d/next/Library/2025-12-01-14-43-58.gh-issue-138122.nRm3ic.rst new file mode 100644 index 00000000000000..e24fea416ff38b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-01-14-43-58.gh-issue-138122.nRm3ic.rst @@ -0,0 +1,5 @@ +The ``_remote_debugging`` module now implements frame caching in the +``RemoteUnwinder`` class to reduce memory reads when profiling remote +processes. When ``cache_frames=True``, unchanged portions of the call stack +are reused from previous samples, significantly improving profiling +performance for deep call stacks. diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index b1582c75bda3e4..1be83b455261ea 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -41,7 +41,7 @@ @MODULE__PICKLE_TRUE@_pickle _pickle.c @MODULE__QUEUE_TRUE@_queue _queuemodule.c @MODULE__RANDOM_TRUE@_random _randommodule.c -@MODULE__REMOTE_DEBUGGING_TRUE@_remote_debugging _remote_debugging/module.c _remote_debugging/object_reading.c _remote_debugging/code_objects.c _remote_debugging/frames.c _remote_debugging/threads.c _remote_debugging/asyncio.c +@MODULE__REMOTE_DEBUGGING_TRUE@_remote_debugging _remote_debugging/module.c _remote_debugging/object_reading.c _remote_debugging/code_objects.c _remote_debugging/frames.c _remote_debugging/frame_cache.c _remote_debugging/threads.c _remote_debugging/asyncio.c @MODULE__STRUCT_TRUE@_struct _struct.c # build supports subinterpreters diff --git a/Modules/_remote_debugging/_remote_debugging.h b/Modules/_remote_debugging/_remote_debugging.h index 70e362ccada6a0..804e2c904e147a 100644 --- a/Modules/_remote_debugging/_remote_debugging.h +++ b/Modules/_remote_debugging/_remote_debugging.h @@ -154,6 +154,39 @@ typedef struct { uintptr_t addr_code_adaptive; } CachedCodeMetadata; +/* Frame cache constants and types */ +#define FRAME_CACHE_MAX_THREADS 32 +#define FRAME_CACHE_MAX_FRAMES 1024 + +typedef struct { + uint64_t thread_id; // 0 = empty slot + uintptr_t addrs[FRAME_CACHE_MAX_FRAMES]; + Py_ssize_t num_addrs; + PyObject *frame_list; // owned reference, NULL if empty +} FrameCacheEntry; + +/* Statistics for profiling performance analysis */ +typedef struct { + uint64_t total_samples; // Total number of get_stack_trace calls + uint64_t frame_cache_hits; // Full cache hits (entire stack unchanged) + uint64_t frame_cache_misses; // Cache misses requiring full walk + uint64_t frame_cache_partial_hits; // Partial hits (stopped at cached frame) + uint64_t frames_read_from_cache; // Total frames retrieved from cache + uint64_t frames_read_from_memory; // Total frames read from remote memory + uint64_t memory_reads; // Total remote memory read operations + uint64_t memory_bytes_read; // Total bytes read from remote memory + uint64_t code_object_cache_hits; // Code object cache hits + uint64_t code_object_cache_misses; // Code object cache misses + uint64_t stale_cache_invalidations; // Times stale entries were cleared +} UnwinderStats; + +/* Stats tracking macros - no-op when stats collection is disabled */ +#define STATS_INC(unwinder, field) \ + do { if ((unwinder)->collect_stats) (unwinder)->stats.field++; } while(0) + +#define STATS_ADD(unwinder, field, val) \ + do { if ((unwinder)->collect_stats) (unwinder)->stats.field += (val); } while(0) + typedef struct { PyTypeObject *RemoteDebugging_Type; PyTypeObject *TaskInfo_Type; @@ -195,7 +228,12 @@ typedef struct { int skip_non_matching_threads; int native; int gc; + int cache_frames; + int collect_stats; // whether to collect statistics + uint32_t stale_invalidation_counter; // counter for throttling frame_cache_invalidate_stale RemoteDebuggingState *cached_state; + FrameCacheEntry *frame_cache; // preallocated array of FRAME_CACHE_MAX_THREADS entries + UnwinderStats stats; // statistics for performance analysis #ifdef Py_GIL_DISABLED uint32_t tlbc_generation; _Py_hashtable_t *tlbc_cache; @@ -363,9 +401,45 @@ extern int process_frame_chain( uintptr_t initial_frame_addr, StackChunkList *chunks, PyObject *frame_info, - uintptr_t gc_frame + uintptr_t gc_frame, + uintptr_t last_profiled_frame, + int *stopped_at_cached_frame, + uintptr_t *frame_addrs, + Py_ssize_t *num_addrs, + Py_ssize_t max_addrs ); +/* Frame cache functions */ +extern int frame_cache_init(RemoteUnwinderObject *unwinder); +extern void frame_cache_cleanup(RemoteUnwinderObject *unwinder); +extern FrameCacheEntry *frame_cache_find(RemoteUnwinderObject *unwinder, uint64_t thread_id); +extern int clear_last_profiled_frames(RemoteUnwinderObject *unwinder); +extern void frame_cache_invalidate_stale(RemoteUnwinderObject *unwinder, PyObject *result); +extern int frame_cache_lookup_and_extend( + RemoteUnwinderObject *unwinder, + uint64_t thread_id, + uintptr_t last_profiled_frame, + PyObject *frame_info, + uintptr_t *frame_addrs, + Py_ssize_t *num_addrs, + Py_ssize_t max_addrs); +// Returns: 1 = stored, 0 = not stored (graceful), -1 = error +extern int frame_cache_store( + RemoteUnwinderObject *unwinder, + uint64_t thread_id, + PyObject *frame_list, + const uintptr_t *addrs, + Py_ssize_t num_addrs); + +extern int collect_frames_with_cache( + RemoteUnwinderObject *unwinder, + uintptr_t frame_addr, + StackChunkList *chunks, + PyObject *frame_info, + uintptr_t gc_frame, + uintptr_t last_profiled_frame, + uint64_t thread_id); + /* ============================================================================ * THREAD FUNCTION DECLARATIONS * ============================================================================ */ diff --git a/Modules/_remote_debugging/clinic/module.c.h b/Modules/_remote_debugging/clinic/module.c.h index 60adb357e32e71..03127b753cc813 100644 --- a/Modules/_remote_debugging/clinic/module.c.h +++ b/Modules/_remote_debugging/clinic/module.c.h @@ -12,7 +12,7 @@ preserve PyDoc_STRVAR(_remote_debugging_RemoteUnwinder___init____doc__, "RemoteUnwinder(pid, *, all_threads=False, only_active_thread=False,\n" " mode=0, debug=False, skip_non_matching_threads=True,\n" -" native=False, gc=False)\n" +" native=False, gc=False, cache_frames=False, stats=False)\n" "--\n" "\n" "Initialize a new RemoteUnwinder object for debugging a remote Python process.\n" @@ -32,6 +32,10 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder___init____doc__, " non-Python code.\n" " gc: If True, include artificial \"\" frames to denote active garbage\n" " collection.\n" +" cache_frames: If True, enable frame caching optimization to avoid re-reading\n" +" unchanged parent frames between samples.\n" +" stats: If True, collect statistics about cache hits, memory reads, etc.\n" +" Use get_stats() to retrieve the collected statistics.\n" "\n" "The RemoteUnwinder provides functionality to inspect and debug a running Python\n" "process, including examining thread states, stack frames and other runtime data.\n" @@ -48,7 +52,8 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, int only_active_thread, int mode, int debug, int skip_non_matching_threads, - int native, int gc); + int native, int gc, + int cache_frames, int stats); static int _remote_debugging_RemoteUnwinder___init__(PyObject *self, PyObject *args, PyObject *kwargs) @@ -56,7 +61,7 @@ _remote_debugging_RemoteUnwinder___init__(PyObject *self, PyObject *args, PyObje int return_value = -1; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 8 + #define NUM_KEYWORDS 10 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -65,7 +70,7 @@ _remote_debugging_RemoteUnwinder___init__(PyObject *self, PyObject *args, PyObje } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(pid), &_Py_ID(all_threads), &_Py_ID(only_active_thread), &_Py_ID(mode), &_Py_ID(debug), &_Py_ID(skip_non_matching_threads), &_Py_ID(native), &_Py_ID(gc), }, + .ob_item = { &_Py_ID(pid), &_Py_ID(all_threads), &_Py_ID(only_active_thread), &_Py_ID(mode), &_Py_ID(debug), &_Py_ID(skip_non_matching_threads), &_Py_ID(native), &_Py_ID(gc), &_Py_ID(cache_frames), &_Py_ID(stats), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -74,14 +79,14 @@ _remote_debugging_RemoteUnwinder___init__(PyObject *self, PyObject *args, PyObje # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"pid", "all_threads", "only_active_thread", "mode", "debug", "skip_non_matching_threads", "native", "gc", NULL}; + static const char * const _keywords[] = {"pid", "all_threads", "only_active_thread", "mode", "debug", "skip_non_matching_threads", "native", "gc", "cache_frames", "stats", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "RemoteUnwinder", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[8]; + PyObject *argsbuf[10]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; @@ -93,6 +98,8 @@ _remote_debugging_RemoteUnwinder___init__(PyObject *self, PyObject *args, PyObje int skip_non_matching_threads = 1; int native = 0; int gc = 0; + int cache_frames = 0; + int stats = 0; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -160,12 +167,30 @@ _remote_debugging_RemoteUnwinder___init__(PyObject *self, PyObject *args, PyObje goto skip_optional_kwonly; } } - gc = PyObject_IsTrue(fastargs[7]); - if (gc < 0) { + if (fastargs[7]) { + gc = PyObject_IsTrue(fastargs[7]); + if (gc < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[8]) { + cache_frames = PyObject_IsTrue(fastargs[8]); + if (cache_frames < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + stats = PyObject_IsTrue(fastargs[9]); + if (stats < 0) { goto exit; } skip_optional_kwonly: - return_value = _remote_debugging_RemoteUnwinder___init___impl((RemoteUnwinderObject *)self, pid, all_threads, only_active_thread, mode, debug, skip_non_matching_threads, native, gc); + return_value = _remote_debugging_RemoteUnwinder___init___impl((RemoteUnwinderObject *)self, pid, all_threads, only_active_thread, mode, debug, skip_non_matching_threads, native, gc, cache_frames, stats); exit: return return_value; @@ -347,4 +372,51 @@ _remote_debugging_RemoteUnwinder_get_async_stack_trace(PyObject *self, PyObject return return_value; } -/*[clinic end generated code: output=99fed5c94cf36881 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_stats__doc__, +"get_stats($self, /)\n" +"--\n" +"\n" +"Get collected statistics about profiling performance.\n" +"\n" +"Returns a dictionary containing statistics about cache performance,\n" +"memory reads, and other profiling metrics. Only available if the\n" +"RemoteUnwinder was created with stats=True.\n" +"\n" +"Returns:\n" +" dict: A dictionary containing:\n" +" - total_samples: Total number of get_stack_trace calls\n" +" - frame_cache_hits: Full cache hits (entire stack unchanged)\n" +" - frame_cache_misses: Cache misses requiring full walk\n" +" - frame_cache_partial_hits: Partial hits (stopped at cached frame)\n" +" - frames_read_from_cache: Total frames retrieved from cache\n" +" - frames_read_from_memory: Total frames read from remote memory\n" +" - memory_reads: Total remote memory read operations\n" +" - memory_bytes_read: Total bytes read from remote memory\n" +" - code_object_cache_hits: Code object cache hits\n" +" - code_object_cache_misses: Code object cache misses\n" +" - stale_cache_invalidations: Times stale cache entries were cleared\n" +" - frame_cache_hit_rate: Percentage of samples that hit the cache\n" +" - code_object_cache_hit_rate: Percentage of code object lookups that hit cache\n" +"\n" +"Raises:\n" +" RuntimeError: If stats collection was not enabled (stats=False)"); + +#define _REMOTE_DEBUGGING_REMOTEUNWINDER_GET_STATS_METHODDEF \ + {"get_stats", (PyCFunction)_remote_debugging_RemoteUnwinder_get_stats, METH_NOARGS, _remote_debugging_RemoteUnwinder_get_stats__doc__}, + +static PyObject * +_remote_debugging_RemoteUnwinder_get_stats_impl(RemoteUnwinderObject *self); + +static PyObject * +_remote_debugging_RemoteUnwinder_get_stats(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _remote_debugging_RemoteUnwinder_get_stats_impl((RemoteUnwinderObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} +/*[clinic end generated code: output=f1fd6c1d4c4c7254 input=a9049054013a1b77]*/ diff --git a/Modules/_remote_debugging/code_objects.c b/Modules/_remote_debugging/code_objects.c index ea3f00c802b110..2cd2505d0f966b 100644 --- a/Modules/_remote_debugging/code_objects.c +++ b/Modules/_remote_debugging/code_objects.c @@ -257,6 +257,11 @@ parse_code_object(RemoteUnwinderObject *unwinder, if (unwinder && unwinder->code_object_cache != NULL) { meta = _Py_hashtable_get(unwinder->code_object_cache, key); + if (meta) { + STATS_INC(unwinder, code_object_cache_hits); + } else { + STATS_INC(unwinder, code_object_cache_misses); + } } if (meta == NULL) { diff --git a/Modules/_remote_debugging/frame_cache.c b/Modules/_remote_debugging/frame_cache.c new file mode 100644 index 00000000000000..4598b9dc353278 --- /dev/null +++ b/Modules/_remote_debugging/frame_cache.c @@ -0,0 +1,236 @@ +/****************************************************************************** + * Remote Debugging Module - Frame Cache + * + * This file contains functions for caching frame information to optimize + * repeated stack unwinding for profiling. + ******************************************************************************/ + +#include "_remote_debugging.h" + +/* ============================================================================ + * FRAME CACHE - stores (address, frame_info) pairs per thread + * Uses preallocated fixed-size arrays for efficiency and bounded memory. + * ============================================================================ */ + +int +frame_cache_init(RemoteUnwinderObject *unwinder) +{ + unwinder->frame_cache = PyMem_Calloc(FRAME_CACHE_MAX_THREADS, sizeof(FrameCacheEntry)); + if (!unwinder->frame_cache) { + PyErr_NoMemory(); + return -1; + } + return 0; +} + +void +frame_cache_cleanup(RemoteUnwinderObject *unwinder) +{ + if (!unwinder->frame_cache) { + return; + } + for (int i = 0; i < FRAME_CACHE_MAX_THREADS; i++) { + Py_CLEAR(unwinder->frame_cache[i].frame_list); + } + PyMem_Free(unwinder->frame_cache); + unwinder->frame_cache = NULL; +} + +// Find cache entry by thread_id +FrameCacheEntry * +frame_cache_find(RemoteUnwinderObject *unwinder, uint64_t thread_id) +{ + if (!unwinder->frame_cache || thread_id == 0) { + return NULL; + } + for (int i = 0; i < FRAME_CACHE_MAX_THREADS; i++) { + if (unwinder->frame_cache[i].thread_id == thread_id) { + return &unwinder->frame_cache[i]; + } + } + return NULL; +} + +// Allocate a cache slot for a thread +// Returns NULL if cache is full (graceful degradation) +static FrameCacheEntry * +frame_cache_alloc_slot(RemoteUnwinderObject *unwinder, uint64_t thread_id) +{ + if (!unwinder->frame_cache || thread_id == 0) { + return NULL; + } + // First check if thread already has an entry + for (int i = 0; i < FRAME_CACHE_MAX_THREADS; i++) { + if (unwinder->frame_cache[i].thread_id == thread_id) { + return &unwinder->frame_cache[i]; + } + } + // Find empty slot + for (int i = 0; i < FRAME_CACHE_MAX_THREADS; i++) { + if (unwinder->frame_cache[i].thread_id == 0) { + return &unwinder->frame_cache[i]; + } + } + // Cache full - graceful degradation + return NULL; +} + +// Remove cache entries for threads not seen in the result +// result structure: list of InterpreterInfo, where InterpreterInfo[1] is threads list, +// and ThreadInfo[0] is the thread_id +void +frame_cache_invalidate_stale(RemoteUnwinderObject *unwinder, PyObject *result) +{ + if (!unwinder->frame_cache || !result || !PyList_Check(result)) { + return; + } + + // Build array of seen thread IDs from result + uint64_t seen_threads[FRAME_CACHE_MAX_THREADS]; + int num_seen = 0; + + Py_ssize_t num_interps = PyList_GET_SIZE(result); + for (Py_ssize_t i = 0; i < num_interps && num_seen < FRAME_CACHE_MAX_THREADS; i++) { + PyObject *interp_info = PyList_GET_ITEM(result, i); + PyObject *threads = PyStructSequence_GetItem(interp_info, 1); + if (!threads || !PyList_Check(threads)) { + continue; + } + Py_ssize_t num_threads = PyList_GET_SIZE(threads); + for (Py_ssize_t j = 0; j < num_threads && num_seen < FRAME_CACHE_MAX_THREADS; j++) { + PyObject *thread_info = PyList_GET_ITEM(threads, j); + PyObject *tid_obj = PyStructSequence_GetItem(thread_info, 0); + if (tid_obj) { + uint64_t tid = PyLong_AsUnsignedLongLong(tid_obj); + if (!PyErr_Occurred()) { + seen_threads[num_seen++] = tid; + } else { + PyErr_Clear(); + } + } + } + } + + // Invalidate entries not in seen list + for (int i = 0; i < FRAME_CACHE_MAX_THREADS; i++) { + if (unwinder->frame_cache[i].thread_id == 0) { + continue; + } + int found = 0; + for (int j = 0; j < num_seen; j++) { + if (unwinder->frame_cache[i].thread_id == seen_threads[j]) { + found = 1; + break; + } + } + if (!found) { + // Clear this entry + Py_CLEAR(unwinder->frame_cache[i].frame_list); + unwinder->frame_cache[i].thread_id = 0; + unwinder->frame_cache[i].num_addrs = 0; + STATS_INC(unwinder, stale_cache_invalidations); + } + } +} + +// Find last_profiled_frame in cache and extend frame_info with cached continuation +// If frame_addrs is provided (not NULL), also extends it with cached addresses +int +frame_cache_lookup_and_extend( + RemoteUnwinderObject *unwinder, + uint64_t thread_id, + uintptr_t last_profiled_frame, + PyObject *frame_info, + uintptr_t *frame_addrs, + Py_ssize_t *num_addrs, + Py_ssize_t max_addrs) +{ + if (!unwinder->frame_cache || last_profiled_frame == 0) { + return 0; + } + + FrameCacheEntry *entry = frame_cache_find(unwinder, thread_id); + if (!entry || !entry->frame_list) { + return 0; + } + + // Find the index where last_profiled_frame matches + Py_ssize_t start_idx = -1; + for (Py_ssize_t i = 0; i < entry->num_addrs; i++) { + if (entry->addrs[i] == last_profiled_frame) { + start_idx = i; + break; + } + } + + if (start_idx < 0) { + return 0; // Not found + } + + Py_ssize_t num_frames = PyList_GET_SIZE(entry->frame_list); + + // Extend frame_info with frames from start_idx onwards + PyObject *slice = PyList_GetSlice(entry->frame_list, start_idx, num_frames); + if (!slice) { + return -1; + } + + Py_ssize_t cur_size = PyList_GET_SIZE(frame_info); + int result = PyList_SetSlice(frame_info, cur_size, cur_size, slice); + Py_DECREF(slice); + + if (result < 0) { + return -1; + } + + // Also extend frame_addrs with cached addresses if provided + if (frame_addrs) { + for (Py_ssize_t i = start_idx; i < entry->num_addrs && *num_addrs < max_addrs; i++) { + frame_addrs[(*num_addrs)++] = entry->addrs[i]; + } + } + + return 1; +} + +// Store frame list with addresses in cache +// Returns: 1 = stored successfully, 0 = not stored (graceful degradation), -1 = error +int +frame_cache_store( + RemoteUnwinderObject *unwinder, + uint64_t thread_id, + PyObject *frame_list, + const uintptr_t *addrs, + Py_ssize_t num_addrs) +{ + if (!unwinder->frame_cache || thread_id == 0) { + return 0; + } + + // Clamp to max frames + if (num_addrs > FRAME_CACHE_MAX_FRAMES) { + num_addrs = FRAME_CACHE_MAX_FRAMES; + } + + FrameCacheEntry *entry = frame_cache_alloc_slot(unwinder, thread_id); + if (!entry) { + // Cache full - graceful degradation + return 0; + } + + // Clear old frame_list if replacing + Py_CLEAR(entry->frame_list); + + // Store full frame list (don't truncate to num_addrs - frames beyond the + // address array limit are still valid and needed for full cache hits) + Py_ssize_t num_frames = PyList_GET_SIZE(frame_list); + entry->frame_list = PyList_GetSlice(frame_list, 0, num_frames); + if (!entry->frame_list) { + return -1; + } + entry->thread_id = thread_id; + memcpy(entry->addrs, addrs, num_addrs * sizeof(uintptr_t)); + entry->num_addrs = num_addrs; + + return 1; +} diff --git a/Modules/_remote_debugging/frames.c b/Modules/_remote_debugging/frames.c index d60caadcb9a11e..b77c0ca556d5b3 100644 --- a/Modules/_remote_debugging/frames.c +++ b/Modules/_remote_debugging/frames.c @@ -189,6 +189,8 @@ parse_frame_object( set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read interpreter frame"); return -1; } + STATS_INC(unwinder, memory_reads); + STATS_ADD(unwinder, memory_bytes_read, SIZEOF_INTERP_FRAME); *previous_frame = GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.previous); uintptr_t code_object = GET_MEMBER_NO_TAG(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.executable); @@ -258,14 +260,39 @@ process_frame_chain( uintptr_t initial_frame_addr, StackChunkList *chunks, PyObject *frame_info, - uintptr_t gc_frame) + uintptr_t gc_frame, + uintptr_t last_profiled_frame, + int *stopped_at_cached_frame, + uintptr_t *frame_addrs, // optional: C array to receive frame addresses + Py_ssize_t *num_addrs, // in/out: current count / updated count + Py_ssize_t max_addrs) // max capacity of frame_addrs array { uintptr_t frame_addr = initial_frame_addr; uintptr_t prev_frame_addr = 0; - const size_t MAX_FRAMES = 1024; + const size_t MAX_FRAMES = 1024 + 512; size_t frame_count = 0; + // Initialize output flag + if (stopped_at_cached_frame) { + *stopped_at_cached_frame = 0; + } + + // Quick check: if current_frame == last_profiled_frame, entire stack is unchanged + if (last_profiled_frame != 0 && initial_frame_addr == last_profiled_frame) { + if (stopped_at_cached_frame) { + *stopped_at_cached_frame = 1; + } + return 0; + } + while ((void*)frame_addr != NULL) { + // Check if we've reached the cached frame - if so, stop here + if (last_profiled_frame != 0 && frame_addr == last_profiled_frame) { + if (stopped_at_cached_frame) { + *stopped_at_cached_frame = 1; + } + break; + } PyObject *frame = NULL; uintptr_t next_frame_addr = 0; uintptr_t stackpointer = 0; @@ -286,7 +313,6 @@ process_frame_chain( } } if (frame == NULL && PyList_GET_SIZE(frame_info) == 0) { - // If the first frame is missing, the chain is broken: const char *e = "Failed to parse initial frame in chain"; PyErr_SetString(PyExc_RuntimeError, e); return -1; @@ -310,36 +336,40 @@ process_frame_chain( extra_frame = &_Py_STR(native); } if (extra_frame) { - // Use "~" as file and 0 as line, since that's what pstats uses: PyObject *extra_frame_info = make_frame_info( unwinder, _Py_LATIN1_CHR('~'), _PyLong_GetZero(), extra_frame); if (extra_frame_info == NULL) { return -1; } - int error = PyList_Append(frame_info, extra_frame_info); - Py_DECREF(extra_frame_info); - if (error) { - const char *e = "Failed to append extra frame to frame info list"; - set_exception_cause(unwinder, PyExc_RuntimeError, e); + if (PyList_Append(frame_info, extra_frame_info) < 0) { + Py_DECREF(extra_frame_info); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append extra frame"); return -1; } + // Extra frames use 0 as address (they're synthetic) + if (frame_addrs && *num_addrs < max_addrs) { + frame_addrs[(*num_addrs)++] = 0; + } + Py_DECREF(extra_frame_info); } if (frame) { if (prev_frame_addr && frame_addr != prev_frame_addr) { const char *f = "Broken frame chain: expected frame at 0x%lx, got 0x%lx"; PyErr_Format(PyExc_RuntimeError, f, prev_frame_addr, frame_addr); Py_DECREF(frame); - const char *e = "Frame chain consistency check failed"; - set_exception_cause(unwinder, PyExc_RuntimeError, e); + set_exception_cause(unwinder, PyExc_RuntimeError, "Frame chain consistency check failed"); return -1; } - if (PyList_Append(frame_info, frame) == -1) { + if (PyList_Append(frame_info, frame) < 0) { Py_DECREF(frame); - const char *e = "Failed to append frame to frame info list"; - set_exception_cause(unwinder, PyExc_RuntimeError, e); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append frame"); return -1; } + // Track the address for this frame + if (frame_addrs && *num_addrs < max_addrs) { + frame_addrs[(*num_addrs)++] = frame_addr; + } Py_DECREF(frame); } @@ -349,3 +379,208 @@ process_frame_chain( return 0; } + +// Clear last_profiled_frame for all threads in the target process. +// This must be called at the start of profiling to avoid stale values +// from previous profilers causing us to stop frame walking early. +int +clear_last_profiled_frames(RemoteUnwinderObject *unwinder) +{ + uintptr_t current_interp = unwinder->interpreter_addr; + uintptr_t zero = 0; + + while (current_interp != 0) { + // Get first thread in this interpreter + uintptr_t tstate_addr; + if (_Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + current_interp + unwinder->debug_offsets.interpreter_state.threads_head, + sizeof(void*), + &tstate_addr) < 0) { + // Non-fatal: just skip clearing + PyErr_Clear(); + return 0; + } + + // Iterate all threads in this interpreter + while (tstate_addr != 0) { + // Clear last_profiled_frame + uintptr_t lpf_addr = tstate_addr + unwinder->debug_offsets.thread_state.last_profiled_frame; + if (_Py_RemoteDebug_WriteRemoteMemory(&unwinder->handle, lpf_addr, + sizeof(uintptr_t), &zero) < 0) { + // Non-fatal: just continue + PyErr_Clear(); + } + + // Move to next thread + if (_Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + tstate_addr + unwinder->debug_offsets.thread_state.next, + sizeof(void*), + &tstate_addr) < 0) { + PyErr_Clear(); + break; + } + } + + // Move to next interpreter + if (_Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + current_interp + unwinder->debug_offsets.interpreter_state.next, + sizeof(void*), + ¤t_interp) < 0) { + PyErr_Clear(); + break; + } + } + + return 0; +} + +// Fast path: check if we have a full cache hit (parent stack unchanged) +// A "full hit" means current frame == last profiled frame, so we can reuse +// cached parent frames. We always read the current frame from memory to get +// updated line numbers (the line within a frame can change between samples). +// Returns: 1 if full hit (frame_info populated with current frame + cached parents), +// 0 if miss, -1 on error +static int +try_full_cache_hit( + RemoteUnwinderObject *unwinder, + uintptr_t frame_addr, + uintptr_t last_profiled_frame, + uint64_t thread_id, + PyObject *frame_info) +{ + if (!unwinder->frame_cache || last_profiled_frame == 0) { + return 0; + } + // Full hit only if current frame == last profiled frame + if (frame_addr != last_profiled_frame) { + return 0; + } + + FrameCacheEntry *entry = frame_cache_find(unwinder, thread_id); + if (!entry || !entry->frame_list) { + return 0; + } + + // Verify first address matches (sanity check) + if (entry->num_addrs == 0 || entry->addrs[0] != frame_addr) { + return 0; + } + + // Always read the current frame from memory to get updated line number + PyObject *current_frame = NULL; + uintptr_t code_object_addr = 0; + uintptr_t previous_frame = 0; + int parse_result = parse_frame_object(unwinder, ¤t_frame, frame_addr, + &code_object_addr, &previous_frame); + if (parse_result < 0) { + return -1; + } + + // Get cached parent frames first (before modifying frame_info) + Py_ssize_t cached_size = PyList_GET_SIZE(entry->frame_list); + PyObject *parent_slice = NULL; + if (cached_size > 1) { + parent_slice = PyList_GetSlice(entry->frame_list, 1, cached_size); + if (!parent_slice) { + Py_XDECREF(current_frame); + return -1; + } + } + + // Now safe to modify frame_info - add current frame if valid + if (current_frame != NULL) { + if (PyList_Append(frame_info, current_frame) < 0) { + Py_DECREF(current_frame); + Py_XDECREF(parent_slice); + return -1; + } + Py_DECREF(current_frame); + STATS_ADD(unwinder, frames_read_from_memory, 1); + } + + // Extend with cached parent frames + if (parent_slice) { + Py_ssize_t cur_size = PyList_GET_SIZE(frame_info); + int result = PyList_SetSlice(frame_info, cur_size, cur_size, parent_slice); + Py_DECREF(parent_slice); + if (result < 0) { + return -1; + } + STATS_ADD(unwinder, frames_read_from_cache, cached_size - 1); + } + + STATS_INC(unwinder, frame_cache_hits); + return 1; +} + +// High-level helper: collect frames with cache optimization +// Returns complete frame_info list, handling all cache logic internally +int +collect_frames_with_cache( + RemoteUnwinderObject *unwinder, + uintptr_t frame_addr, + StackChunkList *chunks, + PyObject *frame_info, + uintptr_t gc_frame, + uintptr_t last_profiled_frame, + uint64_t thread_id) +{ + // Fast path: check for full cache hit first (no allocations needed) + int full_hit = try_full_cache_hit(unwinder, frame_addr, last_profiled_frame, + thread_id, frame_info); + if (full_hit != 0) { + return full_hit < 0 ? -1 : 0; // Either error or success + } + + uintptr_t addrs[FRAME_CACHE_MAX_FRAMES]; + Py_ssize_t num_addrs = 0; + Py_ssize_t frames_before = PyList_GET_SIZE(frame_info); + + int stopped_at_cached = 0; + if (process_frame_chain(unwinder, frame_addr, chunks, frame_info, gc_frame, + last_profiled_frame, &stopped_at_cached, + addrs, &num_addrs, FRAME_CACHE_MAX_FRAMES) < 0) { + return -1; + } + + // Track frames read from memory (frames added by process_frame_chain) + STATS_ADD(unwinder, frames_read_from_memory, PyList_GET_SIZE(frame_info) - frames_before); + + // If stopped at cached frame, extend with cached continuation (both frames and addresses) + if (stopped_at_cached) { + Py_ssize_t frames_before_cache = PyList_GET_SIZE(frame_info); + int cache_result = frame_cache_lookup_and_extend(unwinder, thread_id, last_profiled_frame, + frame_info, addrs, &num_addrs, + FRAME_CACHE_MAX_FRAMES); + if (cache_result < 0) { + return -1; + } + if (cache_result == 0) { + // Cache miss - continue walking from last_profiled_frame to get the rest + STATS_INC(unwinder, frame_cache_misses); + Py_ssize_t frames_before_walk = PyList_GET_SIZE(frame_info); + if (process_frame_chain(unwinder, last_profiled_frame, chunks, frame_info, gc_frame, + 0, NULL, addrs, &num_addrs, FRAME_CACHE_MAX_FRAMES) < 0) { + return -1; + } + STATS_ADD(unwinder, frames_read_from_memory, PyList_GET_SIZE(frame_info) - frames_before_walk); + } else { + // Partial cache hit + STATS_INC(unwinder, frame_cache_partial_hits); + STATS_ADD(unwinder, frames_read_from_cache, PyList_GET_SIZE(frame_info) - frames_before_cache); + } + } else if (last_profiled_frame == 0) { + // No cache involvement (no last_profiled_frame or cache disabled) + STATS_INC(unwinder, frame_cache_misses); + } + + // Store in cache (frame_cache_store handles truncation if num_addrs > FRAME_CACHE_MAX_FRAMES) + if (frame_cache_store(unwinder, thread_id, frame_info, addrs, num_addrs) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_remote_debugging/module.c b/Modules/_remote_debugging/module.c index 6cd9fad37defc7..123e4f5c4d780c 100644 --- a/Modules/_remote_debugging/module.c +++ b/Modules/_remote_debugging/module.c @@ -235,6 +235,8 @@ _remote_debugging.RemoteUnwinder.__init__ skip_non_matching_threads: bool = True native: bool = False gc: bool = False + cache_frames: bool = False + stats: bool = False Initialize a new RemoteUnwinder object for debugging a remote Python process. @@ -253,6 +255,10 @@ Initialize a new RemoteUnwinder object for debugging a remote Python process. non-Python code. gc: If True, include artificial "" frames to denote active garbage collection. + cache_frames: If True, enable frame caching optimization to avoid re-reading + unchanged parent frames between samples. + stats: If True, collect statistics about cache hits, memory reads, etc. + Use get_stats() to retrieve the collected statistics. The RemoteUnwinder provides functionality to inspect and debug a running Python process, including examining thread states, stack frames and other runtime data. @@ -270,8 +276,9 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, int only_active_thread, int mode, int debug, int skip_non_matching_threads, - int native, int gc) -/*[clinic end generated code: output=e9eb6b4df119f6e0 input=606d099059207df2]*/ + int native, int gc, + int cache_frames, int stats) +/*[clinic end generated code: output=b34ef8cce013c975 input=df2221ef114c3d6a]*/ { // Validate that all_threads and only_active_thread are not both True if (all_threads && only_active_thread) { @@ -283,18 +290,24 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, #ifdef Py_GIL_DISABLED if (only_active_thread) { PyErr_SetString(PyExc_ValueError, - "only_active_thread is not supported when Py_GIL_DISABLED is not defined"); + "only_active_thread is not supported in free-threaded builds"); return -1; } #endif self->native = native; self->gc = gc; + self->cache_frames = cache_frames; + self->collect_stats = stats; + self->stale_invalidation_counter = 0; self->debug = debug; self->only_active_thread = only_active_thread; self->mode = mode; self->skip_non_matching_threads = skip_non_matching_threads; self->cached_state = NULL; + self->frame_cache = NULL; + // Initialize stats to zero + memset(&self->stats, 0, sizeof(self->stats)); if (_Py_RemoteDebug_InitProcHandle(&self->handle, pid) < 0) { set_exception_cause(self, PyExc_RuntimeError, "Failed to initialize process handle"); return -1; @@ -375,6 +388,16 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, self->win_process_buffer_size = 0; #endif + if (cache_frames && frame_cache_init(self) < 0) { + return -1; + } + + // Clear stale last_profiled_frame values from previous profilers + // This prevents us from stopping frame walking early due to stale values + if (cache_frames) { + clear_last_profiled_frames(self); + } + return 0; } @@ -429,6 +452,8 @@ static PyObject * _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self) /*[clinic end generated code: output=666192b90c69d567 input=bcff01c73cccc1c0]*/ { + STATS_INC(self, total_samples); + PyObject* result = PyList_New(0); if (!result) { set_exception_cause(self, PyExc_MemoryError, "Failed to create stack trace result list"); @@ -591,7 +616,15 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self } exit: - _Py_RemoteDebug_ClearCache(&self->handle); + // Invalidate cache entries for threads not seen in this sample. + // Only do this every 1024 iterations to avoid performance overhead. + if (self->cache_frames && result) { + if (++self->stale_invalidation_counter >= 1024) { + self->stale_invalidation_counter = 0; + frame_cache_invalidate_stale(self, result); + } + } + _Py_RemoteDebug_ClearCache(&self->handle); return result; } @@ -757,10 +790,114 @@ _remote_debugging_RemoteUnwinder_get_async_stack_trace_impl(RemoteUnwinderObject return NULL; } +/*[clinic input] +@permit_long_docstring_body +@critical_section +_remote_debugging.RemoteUnwinder.get_stats + +Get collected statistics about profiling performance. + +Returns a dictionary containing statistics about cache performance, +memory reads, and other profiling metrics. Only available if the +RemoteUnwinder was created with stats=True. + +Returns: + dict: A dictionary containing: + - total_samples: Total number of get_stack_trace calls + - frame_cache_hits: Full cache hits (entire stack unchanged) + - frame_cache_misses: Cache misses requiring full walk + - frame_cache_partial_hits: Partial hits (stopped at cached frame) + - frames_read_from_cache: Total frames retrieved from cache + - frames_read_from_memory: Total frames read from remote memory + - memory_reads: Total remote memory read operations + - memory_bytes_read: Total bytes read from remote memory + - code_object_cache_hits: Code object cache hits + - code_object_cache_misses: Code object cache misses + - stale_cache_invalidations: Times stale cache entries were cleared + - frame_cache_hit_rate: Percentage of samples that hit the cache + - code_object_cache_hit_rate: Percentage of code object lookups that hit cache + +Raises: + RuntimeError: If stats collection was not enabled (stats=False) +[clinic start generated code]*/ + +static PyObject * +_remote_debugging_RemoteUnwinder_get_stats_impl(RemoteUnwinderObject *self) +/*[clinic end generated code: output=21e36477122be2a0 input=75fef4134c12a8c9]*/ +{ + if (!self->collect_stats) { + PyErr_SetString(PyExc_RuntimeError, + "Statistics collection was not enabled. " + "Create RemoteUnwinder with stats=True to collect statistics."); + return NULL; + } + + PyObject *result = PyDict_New(); + if (!result) { + return NULL; + } + +#define ADD_STAT(name) do { \ + PyObject *val = PyLong_FromUnsignedLongLong(self->stats.name); \ + if (!val || PyDict_SetItemString(result, #name, val) < 0) { \ + Py_XDECREF(val); \ + Py_DECREF(result); \ + return NULL; \ + } \ + Py_DECREF(val); \ +} while(0) + + ADD_STAT(total_samples); + ADD_STAT(frame_cache_hits); + ADD_STAT(frame_cache_misses); + ADD_STAT(frame_cache_partial_hits); + ADD_STAT(frames_read_from_cache); + ADD_STAT(frames_read_from_memory); + ADD_STAT(memory_reads); + ADD_STAT(memory_bytes_read); + ADD_STAT(code_object_cache_hits); + ADD_STAT(code_object_cache_misses); + ADD_STAT(stale_cache_invalidations); + +#undef ADD_STAT + + // Calculate and add derived statistics + // Hit rate is calculated as (hits + partial_hits) / total_cache_lookups + double frame_cache_hit_rate = 0.0; + uint64_t total_cache_lookups = self->stats.frame_cache_hits + self->stats.frame_cache_partial_hits + self->stats.frame_cache_misses; + if (total_cache_lookups > 0) { + frame_cache_hit_rate = 100.0 * (double)(self->stats.frame_cache_hits + self->stats.frame_cache_partial_hits) + / (double)total_cache_lookups; + } + PyObject *hit_rate = PyFloat_FromDouble(frame_cache_hit_rate); + if (!hit_rate || PyDict_SetItemString(result, "frame_cache_hit_rate", hit_rate) < 0) { + Py_XDECREF(hit_rate); + Py_DECREF(result); + return NULL; + } + Py_DECREF(hit_rate); + + double code_object_hit_rate = 0.0; + uint64_t total_code_lookups = self->stats.code_object_cache_hits + self->stats.code_object_cache_misses; + if (total_code_lookups > 0) { + code_object_hit_rate = 100.0 * (double)self->stats.code_object_cache_hits / (double)total_code_lookups; + } + PyObject *code_hit_rate = PyFloat_FromDouble(code_object_hit_rate); + if (!code_hit_rate || PyDict_SetItemString(result, "code_object_cache_hit_rate", code_hit_rate) < 0) { + Py_XDECREF(code_hit_rate); + Py_DECREF(result); + return NULL; + } + Py_DECREF(code_hit_rate); + + return result; +} + static PyMethodDef RemoteUnwinder_methods[] = { _REMOTE_DEBUGGING_REMOTEUNWINDER_GET_STACK_TRACE_METHODDEF _REMOTE_DEBUGGING_REMOTEUNWINDER_GET_ALL_AWAITED_BY_METHODDEF _REMOTE_DEBUGGING_REMOTEUNWINDER_GET_ASYNC_STACK_TRACE_METHODDEF + _REMOTE_DEBUGGING_REMOTEUNWINDER_GET_STATS_METHODDEF {NULL, NULL} }; @@ -787,6 +924,7 @@ RemoteUnwinder_dealloc(PyObject *op) _Py_RemoteDebug_ClearCache(&self->handle); _Py_RemoteDebug_CleanupProcHandle(&self->handle); } + frame_cache_cleanup(self); PyObject_Del(self); Py_DECREF(tp); } diff --git a/Modules/_remote_debugging/threads.c b/Modules/_remote_debugging/threads.c index 99147b01a1b9ed..ce013f902d1ed7 100644 --- a/Modules/_remote_debugging/threads.c +++ b/Modules/_remote_debugging/threads.c @@ -296,6 +296,8 @@ unwind_stack_for_thread( set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read thread state"); goto error; } + STATS_INC(unwinder, memory_reads); + STATS_ADD(unwinder, memory_bytes_read, unwinder->debug_offsets.thread_state.size); long tid = GET_MEMBER(long, ts, unwinder->debug_offsets.thread_state.native_thread_id); @@ -309,6 +311,8 @@ unwind_stack_for_thread( set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read GC state"); goto error; } + STATS_INC(unwinder, memory_reads); + STATS_ADD(unwinder, memory_bytes_read, unwinder->debug_offsets.gc.size); // Calculate thread status using flags (always) int status_flags = 0; @@ -383,14 +387,36 @@ unwind_stack_for_thread( goto error; } - if (copy_stack_chunks(unwinder, *current_tstate, &chunks) < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to copy stack chunks"); - goto error; + // In cache mode, copying stack chunks is more expensive than direct memory reads + if (!unwinder->cache_frames) { + if (copy_stack_chunks(unwinder, *current_tstate, &chunks) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to copy stack chunks"); + goto error; + } } - if (process_frame_chain(unwinder, frame_addr, &chunks, frame_info, gc_frame) < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to process frame chain"); - goto error; + if (unwinder->cache_frames) { + // Use cache to avoid re-reading unchanged parent frames + uintptr_t last_profiled_frame = GET_MEMBER(uintptr_t, ts, + unwinder->debug_offsets.thread_state.last_profiled_frame); + if (collect_frames_with_cache(unwinder, frame_addr, &chunks, frame_info, + gc_frame, last_profiled_frame, tid) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to collect frames"); + goto error; + } + // Update last_profiled_frame for next sample + uintptr_t lpf_addr = *current_tstate + unwinder->debug_offsets.thread_state.last_profiled_frame; + if (_Py_RemoteDebug_WriteRemoteMemory(&unwinder->handle, lpf_addr, + sizeof(uintptr_t), &frame_addr) < 0) { + PyErr_Clear(); // Non-fatal + } + } else { + // No caching - process entire frame chain + if (process_frame_chain(unwinder, frame_addr, &chunks, frame_info, + gc_frame, 0, NULL, NULL, NULL, 0) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to process frame chain"); + goto error; + } } *current_tstate = GET_MEMBER(uintptr_t, ts, unwinder->debug_offsets.thread_state.next); diff --git a/PCbuild/_remote_debugging.vcxproj b/PCbuild/_remote_debugging.vcxproj index 3ef34ef0563eba..c91c9cf3652363 100644 --- a/PCbuild/_remote_debugging.vcxproj +++ b/PCbuild/_remote_debugging.vcxproj @@ -102,6 +102,7 @@ + diff --git a/PCbuild/_remote_debugging.vcxproj.filters b/PCbuild/_remote_debugging.vcxproj.filters index 5c117a79f3bd2d..b37a2c5575c9f5 100644 --- a/PCbuild/_remote_debugging.vcxproj.filters +++ b/PCbuild/_remote_debugging.vcxproj.filters @@ -24,6 +24,9 @@ Source Files + + Source Files + Source Files diff --git a/Python/ceval.c b/Python/ceval.c index aadc6369cbe520..382ae210ebbf2b 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2288,6 +2288,16 @@ clear_gen_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) void _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame * frame) { + // Update last_profiled_frame for remote profiler frame caching. + // By this point, tstate->current_frame is already set to the parent frame. + // Only update if we're popping the exact frame that was last profiled. + // This avoids corrupting the cache when transient frames (called and returned + // between profiler samples) update last_profiled_frame to addresses the + // profiler never saw. + if (tstate->last_profiled_frame != NULL && tstate->last_profiled_frame == frame) { + tstate->last_profiled_frame = tstate->current_frame; + } + if (frame->owner == FRAME_OWNED_BY_THREAD) { clear_thread_frame(tstate, frame); } diff --git a/Python/remote_debug.h b/Python/remote_debug.h index 517568358a0baf..1c02870d3af475 100644 --- a/Python/remote_debug.h +++ b/Python/remote_debug.h @@ -1102,6 +1102,115 @@ _Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address #endif } +#if defined(__linux__) && HAVE_PROCESS_VM_READV +// Fallback write using /proc/pid/mem +static int +_Py_RemoteDebug_WriteRemoteMemoryFallback(proc_handle_t *handle, uintptr_t remote_address, size_t len, const void* src) +{ + if (handle->memfd == -1) { + if (open_proc_mem_fd(handle) < 0) { + return -1; + } + } + + struct iovec local[1]; + Py_ssize_t result = 0; + Py_ssize_t written = 0; + + do { + local[0].iov_base = (char*)src + result; + local[0].iov_len = len - result; + off_t offset = remote_address + result; + + written = pwritev(handle->memfd, local, 1, offset); + if (written < 0) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + + result += written; + } while ((size_t)written != local[0].iov_len); + return 0; +} +#endif // __linux__ + +// Platform-independent memory write function +UNUSED static int +_Py_RemoteDebug_WriteRemoteMemory(proc_handle_t *handle, uintptr_t remote_address, size_t len, const void* src) +{ +#ifdef MS_WINDOWS + SIZE_T written = 0; + SIZE_T result = 0; + do { + if (!WriteProcessMemory(handle->hProcess, (LPVOID)(remote_address + result), (const char*)src + result, len - result, &written)) { + PyErr_SetFromWindowsErr(0); + DWORD error = GetLastError(); + _set_debug_exception_cause(PyExc_OSError, + "WriteProcessMemory failed for PID %d at address 0x%lx " + "(size %zu, partial write %zu bytes): Windows error %lu", + handle->pid, remote_address + result, len - result, result, error); + return -1; + } + result += written; + } while (result < len); + return 0; +#elif defined(__linux__) && HAVE_PROCESS_VM_READV + if (handle->memfd != -1) { + return _Py_RemoteDebug_WriteRemoteMemoryFallback(handle, remote_address, len, src); + } + struct iovec local[1]; + struct iovec remote[1]; + Py_ssize_t result = 0; + Py_ssize_t written = 0; + + do { + local[0].iov_base = (void*)((char*)src + result); + local[0].iov_len = len - result; + remote[0].iov_base = (void*)((char*)remote_address + result); + remote[0].iov_len = len - result; + + written = process_vm_writev(handle->pid, local, 1, remote, 1, 0); + if (written < 0) { + if (errno == ENOSYS) { + return _Py_RemoteDebug_WriteRemoteMemoryFallback(handle, remote_address, len, src); + } + PyErr_SetFromErrno(PyExc_OSError); + _set_debug_exception_cause(PyExc_OSError, + "process_vm_writev failed for PID %d at address 0x%lx " + "(size %zu, partial write %zd bytes): %s", + handle->pid, remote_address + result, len - result, result, strerror(errno)); + return -1; + } + + result += written; + } while ((size_t)written != local[0].iov_len); + return 0; +#elif defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX + kern_return_t kr = mach_vm_write( + handle->task, + (mach_vm_address_t)remote_address, + (vm_offset_t)src, + (mach_msg_type_number_t)len); + + if (kr != KERN_SUCCESS) { + switch (kr) { + case KERN_PROTECTION_FAILURE: + PyErr_SetString(PyExc_PermissionError, "Not enough permissions to write memory"); + break; + case KERN_INVALID_ARGUMENT: + PyErr_SetString(PyExc_PermissionError, "Invalid argument to mach_vm_write"); + break; + default: + PyErr_Format(PyExc_RuntimeError, "Unknown error writing memory: %d", (int)kr); + } + return -1; + } + return 0; +#else + Py_UNREACHABLE(); +#endif +} + UNUSED static int _Py_RemoteDebug_PagedReadRemoteMemory(proc_handle_t *handle, uintptr_t addr, diff --git a/Python/remote_debugging.c b/Python/remote_debugging.c index 71ffb17ed68b1d..5b50b95db94a6d 100644 --- a/Python/remote_debugging.c +++ b/Python/remote_debugging.c @@ -24,104 +24,11 @@ read_memory(proc_handle_t *handle, uintptr_t remote_address, size_t len, void* d return _Py_RemoteDebug_ReadRemoteMemory(handle, remote_address, len, dst); } -// Why is pwritev not guarded? Except on Android API level 23 (no longer -// supported), HAVE_PROCESS_VM_READV is sufficient. -#if defined(__linux__) && HAVE_PROCESS_VM_READV -static int -write_memory_fallback(proc_handle_t *handle, uintptr_t remote_address, size_t len, const void* src) -{ - if (handle->memfd == -1) { - if (open_proc_mem_fd(handle) < 0) { - return -1; - } - } - - struct iovec local[1]; - Py_ssize_t result = 0; - Py_ssize_t written = 0; - - do { - local[0].iov_base = (char*)src + result; - local[0].iov_len = len - result; - off_t offset = remote_address + result; - - written = pwritev(handle->memfd, local, 1, offset); - if (written < 0) { - PyErr_SetFromErrno(PyExc_OSError); - return -1; - } - - result += written; - } while ((size_t)written != local[0].iov_len); - return 0; -} -#endif // __linux__ - +// Use the shared write function from remote_debug.h static int write_memory(proc_handle_t *handle, uintptr_t remote_address, size_t len, const void* src) { -#ifdef MS_WINDOWS - SIZE_T written = 0; - SIZE_T result = 0; - do { - if (!WriteProcessMemory(handle->hProcess, (LPVOID)(remote_address + result), (const char*)src + result, len - result, &written)) { - PyErr_SetFromWindowsErr(0); - return -1; - } - result += written; - } while (result < len); - return 0; -#elif defined(__linux__) && HAVE_PROCESS_VM_READV - if (handle->memfd != -1) { - return write_memory_fallback(handle, remote_address, len, src); - } - struct iovec local[1]; - struct iovec remote[1]; - Py_ssize_t result = 0; - Py_ssize_t written = 0; - - do { - local[0].iov_base = (void*)((char*)src + result); - local[0].iov_len = len - result; - remote[0].iov_base = (void*)((char*)remote_address + result); - remote[0].iov_len = len - result; - - written = process_vm_writev(handle->pid, local, 1, remote, 1, 0); - if (written < 0) { - if (errno == ENOSYS) { - return write_memory_fallback(handle, remote_address, len, src); - } - PyErr_SetFromErrno(PyExc_OSError); - return -1; - } - - result += written; - } while ((size_t)written != local[0].iov_len); - return 0; -#elif defined(__APPLE__) && TARGET_OS_OSX - kern_return_t kr = mach_vm_write( - pid_to_task(handle->pid), - (mach_vm_address_t)remote_address, - (vm_offset_t)src, - (mach_msg_type_number_t)len); - - if (kr != KERN_SUCCESS) { - switch (kr) { - case KERN_PROTECTION_FAILURE: - PyErr_SetString(PyExc_PermissionError, "Not enough permissions to write memory"); - break; - case KERN_INVALID_ARGUMENT: - PyErr_SetString(PyExc_PermissionError, "Invalid argument to mach_vm_write"); - break; - default: - PyErr_Format(PyExc_RuntimeError, "Unknown error writing memory: %d", (int)kr); - } - return -1; - } - return 0; -#else - Py_UNREACHABLE(); -#endif + return _Py_RemoteDebug_WriteRemoteMemory(handle, remote_address, len, src); } static int diff --git a/Tools/inspection/benchmark_external_inspection.py b/Tools/inspection/benchmark_external_inspection.py index 0ac7ac4d385792..9c40c2f4492e58 100644 --- a/Tools/inspection/benchmark_external_inspection.py +++ b/Tools/inspection/benchmark_external_inspection.py @@ -434,7 +434,7 @@ def main(): elif args.threads == "only_active": kwargs["only_active_thread"] = True unwinder = _remote_debugging.RemoteUnwinder( - process.pid, **kwargs + process.pid, cache_frames=True, **kwargs ) results = benchmark(unwinder, duration_seconds=args.duration) finally: From edff5aaa480e459fda995ccbabe99d3fb9cef0b9 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 7 Dec 2025 02:32:20 +0000 Subject: [PATCH 288/638] gh-142368: Refactor test_external_inspection to reduce flakiness in parallel execution (#142369) --- Lib/test/test_external_inspection.py | 3024 ++++++++++++++------------ 1 file changed, 1672 insertions(+), 1352 deletions(-) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 2e6e6eaad06450..f664e8ac53ff95 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -7,6 +7,7 @@ import socket import threading import time +from contextlib import contextmanager from asyncio import staggered, taskgroups, base_events, tasks from unittest.mock import ANY from test.support import ( @@ -27,9 +28,12 @@ PROFILING_MODE_ALL = 3 # Thread status flags -THREAD_STATUS_HAS_GIL = (1 << 0) -THREAD_STATUS_ON_CPU = (1 << 1) -THREAD_STATUS_UNKNOWN = (1 << 2) +THREAD_STATUS_HAS_GIL = 1 << 0 +THREAD_STATUS_ON_CPU = 1 << 1 +THREAD_STATUS_UNKNOWN = 1 << 2 + +# Maximum number of retry attempts for operations that may fail transiently +MAX_TRIES = 10 try: from concurrent import interpreters @@ -48,12 +52,149 @@ ) +# ============================================================================ +# Module-level helper functions +# ============================================================================ + + def _make_test_script(script_dir, script_basename, source): to_return = make_script(script_dir, script_basename, source) importlib.invalidate_caches() return to_return +def _create_server_socket(port, backlog=1): + """Create and configure a server socket for test communication.""" + server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server_socket.bind(("localhost", port)) + server_socket.settimeout(SHORT_TIMEOUT) + server_socket.listen(backlog) + return server_socket + + +def _wait_for_signal(sock, expected_signals, timeout=SHORT_TIMEOUT): + """ + Wait for expected signal(s) from a socket with proper timeout and EOF handling. + + Args: + sock: Connected socket to read from + expected_signals: Single bytes object or list of bytes objects to wait for + timeout: Socket timeout in seconds + + Returns: + bytes: Complete accumulated response buffer + + Raises: + RuntimeError: If connection closed before signal received or timeout + """ + if isinstance(expected_signals, bytes): + expected_signals = [expected_signals] + + sock.settimeout(timeout) + buffer = b"" + + while True: + # Check if all expected signals are in buffer + if all(sig in buffer for sig in expected_signals): + return buffer + + try: + chunk = sock.recv(4096) + if not chunk: + # EOF - connection closed + raise RuntimeError( + f"Connection closed before receiving expected signals. " + f"Expected: {expected_signals}, Got: {buffer[-200:]!r}" + ) + buffer += chunk + except socket.timeout: + raise RuntimeError( + f"Timeout waiting for signals. " + f"Expected: {expected_signals}, Got: {buffer[-200:]!r}" + ) + + +def _wait_for_n_signals(sock, signal_pattern, count, timeout=SHORT_TIMEOUT): + """ + Wait for N occurrences of a signal pattern. + + Args: + sock: Connected socket to read from + signal_pattern: bytes pattern to count (e.g., b"ready") + count: Number of occurrences expected + timeout: Socket timeout in seconds + + Returns: + bytes: Complete accumulated response buffer + + Raises: + RuntimeError: If connection closed or timeout before receiving all signals + """ + sock.settimeout(timeout) + buffer = b"" + found_count = 0 + + while found_count < count: + try: + chunk = sock.recv(4096) + if not chunk: + raise RuntimeError( + f"Connection closed after {found_count}/{count} signals. " + f"Last 200 bytes: {buffer[-200:]!r}" + ) + buffer += chunk + # Count occurrences in entire buffer + found_count = buffer.count(signal_pattern) + except socket.timeout: + raise RuntimeError( + f"Timeout waiting for {count} signals (found {found_count}). " + f"Last 200 bytes: {buffer[-200:]!r}" + ) + + return buffer + + +@contextmanager +def _managed_subprocess(args, timeout=SHORT_TIMEOUT): + """ + Context manager for subprocess lifecycle management. + + Ensures process is properly terminated and cleaned up even on exceptions. + Uses graceful termination first, then forceful kill if needed. + """ + p = subprocess.Popen(args) + try: + yield p + finally: + try: + p.terminate() + try: + p.wait(timeout=timeout) + except subprocess.TimeoutExpired: + p.kill() + try: + p.wait(timeout=timeout) + except subprocess.TimeoutExpired: + pass # Process refuses to die, nothing more we can do + except OSError: + pass # Process already dead + + +def _cleanup_sockets(*sockets): + """Safely close multiple sockets, ignoring errors.""" + for sock in sockets: + if sock is not None: + try: + sock.close() + except OSError: + pass + + +# ============================================================================ +# Decorators and skip conditions +# ============================================================================ + skip_if_not_supported = unittest.skipIf( ( sys.platform != "darwin" @@ -66,40 +207,220 @@ def _make_test_script(script_dir, script_basename, source): def requires_subinterpreters(meth): """Decorator to skip a test if subinterpreters are not supported.""" - return unittest.skipIf(interpreters is None, - 'subinterpreters required')(meth) + return unittest.skipIf(interpreters is None, "subinterpreters required")( + meth + ) + + +# ============================================================================ +# Simple wrapper functions for RemoteUnwinder +# ============================================================================ + +# Errors that can occur transiently when reading process memory without synchronization +RETRIABLE_ERRORS = ( + "Task list appears corrupted", + "Invalid linked list structure reading remote memory", + "Unknown error reading memory", + "Unhandled frame owner", + "Failed to parse initial frame", + "Failed to process frame chain", + "Failed to unwind stack", +) + + +def _is_retriable_error(exc): + """Check if an exception is a transient error that should be retried.""" + msg = str(exc) + return any(msg.startswith(err) or err in msg for err in RETRIABLE_ERRORS) def get_stack_trace(pid): - unwinder = RemoteUnwinder(pid, all_threads=True, debug=True) - return unwinder.get_stack_trace() + for _ in busy_retry(SHORT_TIMEOUT): + try: + unwinder = RemoteUnwinder(pid, all_threads=True, debug=True) + return unwinder.get_stack_trace() + except RuntimeError as e: + if _is_retriable_error(e): + continue + raise + raise RuntimeError("Failed to get stack trace after retries") def get_async_stack_trace(pid): - unwinder = RemoteUnwinder(pid, debug=True) - return unwinder.get_async_stack_trace() + for _ in busy_retry(SHORT_TIMEOUT): + try: + unwinder = RemoteUnwinder(pid, debug=True) + return unwinder.get_async_stack_trace() + except RuntimeError as e: + if _is_retriable_error(e): + continue + raise + raise RuntimeError("Failed to get async stack trace after retries") def get_all_awaited_by(pid): - unwinder = RemoteUnwinder(pid, debug=True) - return unwinder.get_all_awaited_by() + for _ in busy_retry(SHORT_TIMEOUT): + try: + unwinder = RemoteUnwinder(pid, debug=True) + return unwinder.get_all_awaited_by() + except RuntimeError as e: + if _is_retriable_error(e): + continue + raise + raise RuntimeError("Failed to get all awaited_by after retries") + + +# ============================================================================ +# Base test class with shared infrastructure +# ============================================================================ -class TestGetStackTrace(unittest.TestCase): +class RemoteInspectionTestBase(unittest.TestCase): + """Base class for remote inspection tests with common helpers.""" + maxDiff = None + def _run_script_and_get_trace( + self, + script, + trace_func, + wait_for_signals=None, + port=None, + backlog=1, + ): + """ + Common pattern: run a script, wait for signals, get trace. + + Args: + script: Script content (will be formatted with port if {port} present) + trace_func: Function to call with pid to get trace (e.g., get_stack_trace) + wait_for_signals: Signal(s) to wait for before getting trace + port: Port to use (auto-selected if None) + backlog: Socket listen backlog + + Returns: + tuple: (trace_result, script_name) + """ + if port is None: + port = find_unused_port() + + # Format script with port if needed + if "{port}" in script or "{{port}}" in script: + script = script.replace("{{port}}", "{port}").format(port=port) + + with os_helper.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + + server_socket = _create_server_socket(port, backlog) + script_name = _make_test_script(script_dir, "script", script) + client_socket = None + + try: + with _managed_subprocess([sys.executable, script_name]) as p: + client_socket, _ = server_socket.accept() + server_socket.close() + server_socket = None + + if wait_for_signals: + _wait_for_signal(client_socket, wait_for_signals) + + try: + trace = trace_func(p.pid) + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" + ) + return trace, script_name + finally: + _cleanup_sockets(client_socket, server_socket) + + def _find_frame_in_trace(self, stack_trace, predicate): + """ + Find a frame matching predicate in stack trace. + + Args: + stack_trace: List of InterpreterInfo objects + predicate: Function(frame) -> bool + + Returns: + FrameInfo or None + """ + for interpreter_info in stack_trace: + for thread_info in interpreter_info.threads: + for frame in thread_info.frame_info: + if predicate(frame): + return frame + return None + + def _find_thread_by_id(self, stack_trace, thread_id): + """Find a thread by its native thread ID.""" + for interpreter_info in stack_trace: + for thread_info in interpreter_info.threads: + if thread_info.thread_id == thread_id: + return thread_info + return None + + def _find_thread_with_frame(self, stack_trace, frame_predicate): + """Find a thread containing a frame matching predicate.""" + for interpreter_info in stack_trace: + for thread_info in interpreter_info.threads: + for frame in thread_info.frame_info: + if frame_predicate(frame): + return thread_info + return None + + def _get_thread_statuses(self, stack_trace): + """Extract thread_id -> status mapping from stack trace.""" + statuses = {} + for interpreter_info in stack_trace: + for thread_info in interpreter_info.threads: + statuses[thread_info.thread_id] = thread_info.status + return statuses + + def _get_task_id_map(self, stack_trace): + """Create task_id -> task mapping from async stack trace.""" + return {task.task_id: task for task in stack_trace[0].awaited_by} + + def _get_awaited_by_relationships(self, stack_trace): + """Extract task name to awaited_by set mapping.""" + id_to_task = self._get_task_id_map(stack_trace) + return { + task.task_name: set( + id_to_task[awaited.task_name].task_name + for awaited in task.awaited_by + ) + for task in stack_trace[0].awaited_by + } + + def _extract_coroutine_stacks(self, stack_trace): + """Extract and format coroutine stacks from tasks.""" + return { + task.task_name: sorted( + tuple(tuple(frame) for frame in coro.call_stack) + for coro in task.coroutine_stack + ) + for task in stack_trace[0].awaited_by + } + + +# ============================================================================ +# Test classes +# ============================================================================ + + +class TestGetStackTrace(RemoteInspectionTestBase): @skip_if_not_supported @unittest.skipIf( sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support", ) def test_remote_stack_trace(self): - # Spawn a process with some realistic Python code port = find_unused_port() script = textwrap.dedent( f"""\ import time, sys, socket, threading - # Connect to the test process + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect(('localhost', {port})) @@ -112,80 +433,78 @@ def baz(): foo() def foo(): - sock.sendall(b"ready:thread\\n"); time.sleep(10_000) # same line number + sock.sendall(b"ready:thread\\n"); time.sleep(10_000) t = threading.Thread(target=bar) t.start() - sock.sendall(b"ready:main\\n"); t.join() # same line number + sock.sendall(b"ready:main\\n"); t.join() """ ) - stack_trace = None + with os_helper.temp_dir() as work_dir: script_dir = os.path.join(work_dir, "script_pkg") os.mkdir(script_dir) - # Create a socket server to communicate with the target process - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server_socket.bind(("localhost", port)) - server_socket.settimeout(SHORT_TIMEOUT) - server_socket.listen(1) - + server_socket = _create_server_socket(port) script_name = _make_test_script(script_dir, "script", script) client_socket = None + try: - p = subprocess.Popen([sys.executable, script_name]) - client_socket, _ = server_socket.accept() - server_socket.close() - response = b"" - while ( - b"ready:main" not in response - or b"ready:thread" not in response - ): - response += client_socket.recv(1024) - stack_trace = get_stack_trace(p.pid) - except PermissionError: - self.skipTest( - "Insufficient permissions to read the stack trace" - ) + with _managed_subprocess([sys.executable, script_name]) as p: + client_socket, _ = server_socket.accept() + server_socket.close() + server_socket = None + + _wait_for_signal( + client_socket, [b"ready:main", b"ready:thread"] + ) + + try: + stack_trace = get_stack_trace(p.pid) + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" + ) + + thread_expected_stack_trace = [ + FrameInfo([script_name, 15, "foo"]), + FrameInfo([script_name, 12, "baz"]), + FrameInfo([script_name, 9, "bar"]), + FrameInfo([threading.__file__, ANY, "Thread.run"]), + FrameInfo( + [ + threading.__file__, + ANY, + "Thread._bootstrap_inner", + ] + ), + FrameInfo( + [threading.__file__, ANY, "Thread._bootstrap"] + ), + ] + + # Find expected thread stack + found_thread = self._find_thread_with_frame( + stack_trace, + lambda f: f.funcname == "foo" and f.lineno == 15, + ) + self.assertIsNotNone( + found_thread, "Expected thread stack trace not found" + ) + self.assertEqual( + found_thread.frame_info, thread_expected_stack_trace + ) + + # Check main thread + main_frame = FrameInfo([script_name, 19, ""]) + found_main = self._find_frame_in_trace( + stack_trace, lambda f: f == main_frame + ) + self.assertIsNotNone( + found_main, "Main thread stack trace not found" + ) finally: - if client_socket is not None: - client_socket.close() - p.kill() - p.terminate() - p.wait(timeout=SHORT_TIMEOUT) - - thread_expected_stack_trace = [ - FrameInfo([script_name, 15, "foo"]), - FrameInfo([script_name, 12, "baz"]), - FrameInfo([script_name, 9, "bar"]), - FrameInfo([threading.__file__, ANY, "Thread.run"]), - FrameInfo([threading.__file__, ANY, "Thread._bootstrap_inner"]), - FrameInfo([threading.__file__, ANY, "Thread._bootstrap"]), - ] - # Is possible that there are more threads, so we check that the - # expected stack traces are in the result (looking at you Windows!) - found_expected_stack = False - for interpreter_info in stack_trace: - for thread_info in interpreter_info.threads: - if thread_info.frame_info == thread_expected_stack_trace: - found_expected_stack = True - break - if found_expected_stack: - break - self.assertTrue(found_expected_stack, "Expected thread stack trace not found") - - # Check that the main thread stack trace is in the result - frame = FrameInfo([script_name, 19, ""]) - main_thread_found = False - for interpreter_info in stack_trace: - for thread_info in interpreter_info.threads: - if frame in thread_info.frame_info: - main_thread_found = True - break - if main_thread_found: - break - self.assertTrue(main_thread_found, "Main thread stack trace not found in result") + _cleanup_sockets(client_socket, server_socket) @skip_if_not_supported @unittest.skipIf( @@ -193,7 +512,6 @@ def foo(): "Test only runs on Linux with process_vm_readv support", ) def test_async_remote_stack_trace(self): - # Spawn a process with some realistic Python code port = find_unused_port() script = textwrap.dedent( f"""\ @@ -201,12 +519,12 @@ def test_async_remote_stack_trace(self): import time import sys import socket - # Connect to the test process + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect(('localhost', {port})) def c5(): - sock.sendall(b"ready"); time.sleep(10_000) # same line number + sock.sendall(b"ready"); time.sleep(10_000) async def c4(): await asyncio.sleep(0) @@ -237,7 +555,7 @@ def new_eager_loop(): asyncio.run(main(), loop_factory={{TASK_FACTORY}}) """ ) - stack_trace = None + for task_factory_variant in "asyncio.new_event_loop", "new_eager_loop": with ( self.subTest(task_factory_variant=task_factory_variant), @@ -245,195 +563,203 @@ def new_eager_loop(): ): script_dir = os.path.join(work_dir, "script_pkg") os.mkdir(script_dir) - server_socket = socket.socket( - socket.AF_INET, socket.SOCK_STREAM - ) - server_socket.setsockopt( - socket.SOL_SOCKET, socket.SO_REUSEADDR, 1 - ) - server_socket.bind(("localhost", port)) - server_socket.settimeout(SHORT_TIMEOUT) - server_socket.listen(1) + + server_socket = _create_server_socket(port) script_name = _make_test_script( script_dir, "script", script.format(TASK_FACTORY=task_factory_variant), ) client_socket = None + try: - p = subprocess.Popen([sys.executable, script_name]) - client_socket, _ = server_socket.accept() - server_socket.close() - response = client_socket.recv(1024) - self.assertEqual(response, b"ready") - stack_trace = get_async_stack_trace(p.pid) - except PermissionError: - self.skipTest( - "Insufficient permissions to read the stack trace" - ) - finally: - if client_socket is not None: - client_socket.close() - p.kill() - p.terminate() - p.wait(timeout=SHORT_TIMEOUT) - - # First check all the tasks are present - tasks_names = [ - task.task_name for task in stack_trace[0].awaited_by - ] - for task_name in ["c2_root", "sub_main_1", "sub_main_2"]: - self.assertIn(task_name, tasks_names) - - # Now ensure that the awaited_by_relationships are correct - id_to_task = { - task.task_id: task for task in stack_trace[0].awaited_by - } - task_name_to_awaited_by = { - task.task_name: set( - id_to_task[awaited.task_name].task_name - for awaited in task.awaited_by - ) - for task in stack_trace[0].awaited_by - } - self.assertEqual( - task_name_to_awaited_by, - { - "c2_root": {"Task-1", "sub_main_1", "sub_main_2"}, - "Task-1": set(), - "sub_main_1": {"Task-1"}, - "sub_main_2": {"Task-1"}, - }, - ) + with _managed_subprocess( + [sys.executable, script_name] + ) as p: + client_socket, _ = server_socket.accept() + server_socket.close() + server_socket = None - # Now ensure that the coroutine stacks are correct - coroutine_stacks = { - task.task_name: sorted( - tuple(tuple(frame) for frame in coro.call_stack) - for coro in task.coroutine_stack - ) - for task in stack_trace[0].awaited_by - } - self.assertEqual( - coroutine_stacks, - { - "Task-1": [ - ( - tuple( - [ - taskgroups.__file__, - ANY, - "TaskGroup._aexit", - ] - ), - tuple( - [ - taskgroups.__file__, - ANY, - "TaskGroup.__aexit__", - ] - ), - tuple([script_name, 26, "main"]), - ) - ], - "c2_root": [ - ( - tuple([script_name, 10, "c5"]), - tuple([script_name, 14, "c4"]), - tuple([script_name, 17, "c3"]), - tuple([script_name, 20, "c2"]), + response = _wait_for_signal(client_socket, b"ready") + self.assertIn(b"ready", response) + + try: + stack_trace = get_async_stack_trace(p.pid) + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" ) - ], - "sub_main_1": [(tuple([script_name, 23, "c1"]),)], - "sub_main_2": [(tuple([script_name, 23, "c1"]),)], - }, - ) - # Now ensure the coroutine stacks for the awaited_by relationships are correct. - awaited_by_coroutine_stacks = { - task.task_name: sorted( - ( - id_to_task[coro.task_name].task_name, - tuple(tuple(frame) for frame in coro.call_stack), + # Check all tasks are present + tasks_names = [ + task.task_name + for task in stack_trace[0].awaited_by + ] + for task_name in [ + "c2_root", + "sub_main_1", + "sub_main_2", + ]: + self.assertIn(task_name, tasks_names) + + # Check awaited_by relationships + relationships = self._get_awaited_by_relationships( + stack_trace ) - for coro in task.awaited_by - ) - for task in stack_trace[0].awaited_by - } - self.assertEqual( - awaited_by_coroutine_stacks, - { - "Task-1": [], - "c2_root": [ - ( - "Task-1", - ( - tuple( - [ - taskgroups.__file__, - ANY, - "TaskGroup._aexit", - ] - ), - tuple( - [ - taskgroups.__file__, - ANY, - "TaskGroup.__aexit__", - ] - ), - tuple([script_name, 26, "main"]), - ), - ), - ("sub_main_1", (tuple([script_name, 23, "c1"]),)), - ("sub_main_2", (tuple([script_name, 23, "c1"]),)), - ], - "sub_main_1": [ - ( - "Task-1", + self.assertEqual( + relationships, + { + "c2_root": { + "Task-1", + "sub_main_1", + "sub_main_2", + }, + "Task-1": set(), + "sub_main_1": {"Task-1"}, + "sub_main_2": {"Task-1"}, + }, + ) + + # Check coroutine stacks + coroutine_stacks = self._extract_coroutine_stacks( + stack_trace + ) + self.assertEqual( + coroutine_stacks, + { + "Task-1": [ + ( + tuple( + [ + taskgroups.__file__, + ANY, + "TaskGroup._aexit", + ] + ), + tuple( + [ + taskgroups.__file__, + ANY, + "TaskGroup.__aexit__", + ] + ), + tuple([script_name, 26, "main"]), + ) + ], + "c2_root": [ + ( + tuple([script_name, 10, "c5"]), + tuple([script_name, 14, "c4"]), + tuple([script_name, 17, "c3"]), + tuple([script_name, 20, "c2"]), + ) + ], + "sub_main_1": [ + (tuple([script_name, 23, "c1"]),) + ], + "sub_main_2": [ + (tuple([script_name, 23, "c1"]),) + ], + }, + ) + + # Check awaited_by coroutine stacks + id_to_task = self._get_task_id_map(stack_trace) + awaited_by_coroutine_stacks = { + task.task_name: sorted( ( + id_to_task[coro.task_name].task_name, tuple( - [ - taskgroups.__file__, - ANY, - "TaskGroup._aexit", - ] - ), - tuple( - [ - taskgroups.__file__, - ANY, - "TaskGroup.__aexit__", - ] + tuple(frame) + for frame in coro.call_stack ), - tuple([script_name, 26, "main"]), - ), + ) + for coro in task.awaited_by ) - ], - "sub_main_2": [ - ( - "Task-1", - ( - tuple( - [ - taskgroups.__file__, - ANY, - "TaskGroup._aexit", - ] + for task in stack_trace[0].awaited_by + } + self.assertEqual( + awaited_by_coroutine_stacks, + { + "Task-1": [], + "c2_root": [ + ( + "Task-1", + ( + tuple( + [ + taskgroups.__file__, + ANY, + "TaskGroup._aexit", + ] + ), + tuple( + [ + taskgroups.__file__, + ANY, + "TaskGroup.__aexit__", + ] + ), + tuple([script_name, 26, "main"]), + ), ), - tuple( - [ - taskgroups.__file__, - ANY, - "TaskGroup.__aexit__", - ] + ( + "sub_main_1", + (tuple([script_name, 23, "c1"]),), ), - tuple([script_name, 26, "main"]), - ), - ) - ], - }, - ) + ( + "sub_main_2", + (tuple([script_name, 23, "c1"]),), + ), + ], + "sub_main_1": [ + ( + "Task-1", + ( + tuple( + [ + taskgroups.__file__, + ANY, + "TaskGroup._aexit", + ] + ), + tuple( + [ + taskgroups.__file__, + ANY, + "TaskGroup.__aexit__", + ] + ), + tuple([script_name, 26, "main"]), + ), + ) + ], + "sub_main_2": [ + ( + "Task-1", + ( + tuple( + [ + taskgroups.__file__, + ANY, + "TaskGroup._aexit", + ] + ), + tuple( + [ + taskgroups.__file__, + ANY, + "TaskGroup.__aexit__", + ] + ), + tuple([script_name, 26, "main"]), + ), + ) + ], + }, + ) + finally: + _cleanup_sockets(client_socket, server_socket) @skip_if_not_supported @unittest.skipIf( @@ -441,7 +767,6 @@ def new_eager_loop(): "Test only runs on Linux with process_vm_readv support", ) def test_asyncgen_remote_stack_trace(self): - # Spawn a process with some realistic Python code port = find_unused_port() script = textwrap.dedent( f"""\ @@ -449,12 +774,12 @@ def test_asyncgen_remote_stack_trace(self): import time import sys import socket - # Connect to the test process + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect(('localhost', {port})) async def gen_nested_call(): - sock.sendall(b"ready"); time.sleep(10_000) # same line number + sock.sendall(b"ready"); time.sleep(10_000) async def gen(): for num in range(2): @@ -469,59 +794,56 @@ async def main(): asyncio.run(main()) """ ) - stack_trace = None + with os_helper.temp_dir() as work_dir: script_dir = os.path.join(work_dir, "script_pkg") os.mkdir(script_dir) - # Create a socket server to communicate with the target process - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server_socket.bind(("localhost", port)) - server_socket.settimeout(SHORT_TIMEOUT) - server_socket.listen(1) + + server_socket = _create_server_socket(port) script_name = _make_test_script(script_dir, "script", script) client_socket = None + try: - p = subprocess.Popen([sys.executable, script_name]) - client_socket, _ = server_socket.accept() - server_socket.close() - response = client_socket.recv(1024) - self.assertEqual(response, b"ready") - stack_trace = get_async_stack_trace(p.pid) - except PermissionError: - self.skipTest( - "Insufficient permissions to read the stack trace" - ) - finally: - if client_socket is not None: - client_socket.close() - p.kill() - p.terminate() - p.wait(timeout=SHORT_TIMEOUT) + with _managed_subprocess([sys.executable, script_name]) as p: + client_socket, _ = server_socket.accept() + server_socket.close() + server_socket = None - # For this simple asyncgen test, we only expect one task with the full coroutine stack - self.assertEqual(len(stack_trace[0].awaited_by), 1) - task = stack_trace[0].awaited_by[0] - self.assertEqual(task.task_name, "Task-1") + response = _wait_for_signal(client_socket, b"ready") + self.assertIn(b"ready", response) - # Check the coroutine stack - based on actual output, only shows main - coroutine_stack = sorted( - tuple(tuple(frame) for frame in coro.call_stack) - for coro in task.coroutine_stack - ) - self.assertEqual( - coroutine_stack, - [ - ( - tuple([script_name, 10, "gen_nested_call"]), - tuple([script_name, 16, "gen"]), - tuple([script_name, 19, "main"]), + try: + stack_trace = get_async_stack_trace(p.pid) + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" + ) + + # For this simple asyncgen test, we only expect one task + self.assertEqual(len(stack_trace[0].awaited_by), 1) + task = stack_trace[0].awaited_by[0] + self.assertEqual(task.task_name, "Task-1") + + # Check the coroutine stack + coroutine_stack = sorted( + tuple(tuple(frame) for frame in coro.call_stack) + for coro in task.coroutine_stack + ) + self.assertEqual( + coroutine_stack, + [ + ( + tuple([script_name, 10, "gen_nested_call"]), + tuple([script_name, 16, "gen"]), + tuple([script_name, 19, "main"]), + ) + ], ) - ], - ) - # No awaited_by relationships expected for this simple case - self.assertEqual(task.awaited_by, []) + # No awaited_by relationships expected + self.assertEqual(task.awaited_by, []) + finally: + _cleanup_sockets(client_socket, server_socket) @skip_if_not_supported @unittest.skipIf( @@ -529,7 +851,6 @@ async def main(): "Test only runs on Linux with process_vm_readv support", ) def test_async_gather_remote_stack_trace(self): - # Spawn a process with some realistic Python code port = find_unused_port() script = textwrap.dedent( f"""\ @@ -537,13 +858,13 @@ def test_async_gather_remote_stack_trace(self): import time import sys import socket - # Connect to the test process + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect(('localhost', {port})) async def deep(): await asyncio.sleep(0) - sock.sendall(b"ready"); time.sleep(10_000) # same line number + sock.sendall(b"ready"); time.sleep(10_000) async def c1(): await asyncio.sleep(0) @@ -558,103 +879,92 @@ async def main(): asyncio.run(main()) """ ) - stack_trace = None + with os_helper.temp_dir() as work_dir: - script_dir = os.path.join(work_dir, "script_pkg") - os.mkdir(script_dir) - # Create a socket server to communicate with the target process - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server_socket.bind(("localhost", port)) - server_socket.settimeout(SHORT_TIMEOUT) - server_socket.listen(1) - script_name = _make_test_script(script_dir, "script", script) - client_socket = None - try: - p = subprocess.Popen([sys.executable, script_name]) - client_socket, _ = server_socket.accept() - server_socket.close() - response = client_socket.recv(1024) - self.assertEqual(response, b"ready") - stack_trace = get_async_stack_trace(p.pid) - except PermissionError: - self.skipTest( - "Insufficient permissions to read the stack trace" - ) - finally: - if client_socket is not None: - client_socket.close() - p.kill() - p.terminate() - p.wait(timeout=SHORT_TIMEOUT) - - # First check all the tasks are present - tasks_names = [ - task.task_name for task in stack_trace[0].awaited_by - ] - for task_name in ["Task-1", "Task-2"]: - self.assertIn(task_name, tasks_names) - - # Now ensure that the awaited_by_relationships are correct - id_to_task = { - task.task_id: task for task in stack_trace[0].awaited_by - } - task_name_to_awaited_by = { - task.task_name: set( - id_to_task[awaited.task_name].task_name - for awaited in task.awaited_by - ) - for task in stack_trace[0].awaited_by - } - self.assertEqual( - task_name_to_awaited_by, - { - "Task-1": set(), - "Task-2": {"Task-1"}, - }, - ) + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) - # Now ensure that the coroutine stacks are correct - coroutine_stacks = { - task.task_name: sorted( - tuple(tuple(frame) for frame in coro.call_stack) - for coro in task.coroutine_stack - ) - for task in stack_trace[0].awaited_by - } - self.assertEqual( - coroutine_stacks, - { - "Task-1": [(tuple([script_name, 21, "main"]),)], - "Task-2": [ - ( - tuple([script_name, 11, "deep"]), - tuple([script_name, 15, "c1"]), + server_socket = _create_server_socket(port) + script_name = _make_test_script(script_dir, "script", script) + client_socket = None + + try: + with _managed_subprocess([sys.executable, script_name]) as p: + client_socket, _ = server_socket.accept() + server_socket.close() + server_socket = None + + response = _wait_for_signal(client_socket, b"ready") + self.assertIn(b"ready", response) + + try: + stack_trace = get_async_stack_trace(p.pid) + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" ) - ], - }, - ) - # Now ensure the coroutine stacks for the awaited_by relationships are correct. - awaited_by_coroutine_stacks = { - task.task_name: sorted( - ( - id_to_task[coro.task_name].task_name, - tuple(tuple(frame) for frame in coro.call_stack), + # Check all tasks are present + tasks_names = [ + task.task_name for task in stack_trace[0].awaited_by + ] + for task_name in ["Task-1", "Task-2"]: + self.assertIn(task_name, tasks_names) + + # Check awaited_by relationships + relationships = self._get_awaited_by_relationships( + stack_trace ) - for coro in task.awaited_by - ) - for task in stack_trace[0].awaited_by - } - self.assertEqual( - awaited_by_coroutine_stacks, - { - "Task-1": [], - "Task-2": [ - ("Task-1", (tuple([script_name, 21, "main"]),)) - ], - }, - ) + self.assertEqual( + relationships, + { + "Task-1": set(), + "Task-2": {"Task-1"}, + }, + ) + + # Check coroutine stacks + coroutine_stacks = self._extract_coroutine_stacks( + stack_trace + ) + self.assertEqual( + coroutine_stacks, + { + "Task-1": [(tuple([script_name, 21, "main"]),)], + "Task-2": [ + ( + tuple([script_name, 11, "deep"]), + tuple([script_name, 15, "c1"]), + ) + ], + }, + ) + + # Check awaited_by coroutine stacks + id_to_task = self._get_task_id_map(stack_trace) + awaited_by_coroutine_stacks = { + task.task_name: sorted( + ( + id_to_task[coro.task_name].task_name, + tuple( + tuple(frame) for frame in coro.call_stack + ), + ) + for coro in task.awaited_by + ) + for task in stack_trace[0].awaited_by + } + self.assertEqual( + awaited_by_coroutine_stacks, + { + "Task-1": [], + "Task-2": [ + ("Task-1", (tuple([script_name, 21, "main"]),)) + ], + }, + ) + finally: + _cleanup_sockets(client_socket, server_socket) @skip_if_not_supported @unittest.skipIf( @@ -662,7 +972,6 @@ async def main(): "Test only runs on Linux with process_vm_readv support", ) def test_async_staggered_race_remote_stack_trace(self): - # Spawn a process with some realistic Python code port = find_unused_port() script = textwrap.dedent( f"""\ @@ -670,13 +979,13 @@ def test_async_staggered_race_remote_stack_trace(self): import time import sys import socket - # Connect to the test process + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect(('localhost', {port})) async def deep(): await asyncio.sleep(0) - sock.sendall(b"ready"); time.sleep(10_000) # same line number + sock.sendall(b"ready"); time.sleep(10_000) async def c1(): await asyncio.sleep(0) @@ -694,123 +1003,122 @@ async def main(): asyncio.run(main()) """ ) - stack_trace = None + with os_helper.temp_dir() as work_dir: script_dir = os.path.join(work_dir, "script_pkg") os.mkdir(script_dir) - # Create a socket server to communicate with the target process - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server_socket.bind(("localhost", port)) - server_socket.settimeout(SHORT_TIMEOUT) - server_socket.listen(1) + + server_socket = _create_server_socket(port) script_name = _make_test_script(script_dir, "script", script) client_socket = None + try: - p = subprocess.Popen([sys.executable, script_name]) - client_socket, _ = server_socket.accept() - server_socket.close() - response = client_socket.recv(1024) - self.assertEqual(response, b"ready") - stack_trace = get_async_stack_trace(p.pid) - except PermissionError: - self.skipTest( - "Insufficient permissions to read the stack trace" - ) - finally: - if client_socket is not None: - client_socket.close() - p.kill() - p.terminate() - p.wait(timeout=SHORT_TIMEOUT) - - # First check all the tasks are present - tasks_names = [ - task.task_name for task in stack_trace[0].awaited_by - ] - for task_name in ["Task-1", "Task-2"]: - self.assertIn(task_name, tasks_names) - - # Now ensure that the awaited_by_relationships are correct - id_to_task = { - task.task_id: task for task in stack_trace[0].awaited_by - } - task_name_to_awaited_by = { - task.task_name: set( - id_to_task[awaited.task_name].task_name - for awaited in task.awaited_by - ) - for task in stack_trace[0].awaited_by - } - self.assertEqual( - task_name_to_awaited_by, - { - "Task-1": set(), - "Task-2": {"Task-1"}, - }, - ) + with _managed_subprocess([sys.executable, script_name]) as p: + client_socket, _ = server_socket.accept() + server_socket.close() + server_socket = None - # Now ensure that the coroutine stacks are correct - coroutine_stacks = { - task.task_name: sorted( - tuple(tuple(frame) for frame in coro.call_stack) - for coro in task.coroutine_stack - ) - for task in stack_trace[0].awaited_by - } - self.assertEqual( - coroutine_stacks, - { - "Task-1": [ - ( - tuple([staggered.__file__, ANY, "staggered_race"]), - tuple([script_name, 21, "main"]), - ) - ], - "Task-2": [ - ( - tuple([script_name, 11, "deep"]), - tuple([script_name, 15, "c1"]), - tuple( - [ - staggered.__file__, - ANY, - "staggered_race..run_one_coro", - ] - ), + response = _wait_for_signal(client_socket, b"ready") + self.assertIn(b"ready", response) + + try: + stack_trace = get_async_stack_trace(p.pid) + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" ) - ], - }, - ) - # Now ensure the coroutine stacks for the awaited_by relationships are correct. - awaited_by_coroutine_stacks = { - task.task_name: sorted( - ( - id_to_task[coro.task_name].task_name, - tuple(tuple(frame) for frame in coro.call_stack), + # Check all tasks are present + tasks_names = [ + task.task_name for task in stack_trace[0].awaited_by + ] + for task_name in ["Task-1", "Task-2"]: + self.assertIn(task_name, tasks_names) + + # Check awaited_by relationships + relationships = self._get_awaited_by_relationships( + stack_trace ) - for coro in task.awaited_by - ) - for task in stack_trace[0].awaited_by - } - self.assertEqual( - awaited_by_coroutine_stacks, - { - "Task-1": [], - "Task-2": [ - ( - "Task-1", + self.assertEqual( + relationships, + { + "Task-1": set(), + "Task-2": {"Task-1"}, + }, + ) + + # Check coroutine stacks + coroutine_stacks = self._extract_coroutine_stacks( + stack_trace + ) + self.assertEqual( + coroutine_stacks, + { + "Task-1": [ + ( + tuple( + [ + staggered.__file__, + ANY, + "staggered_race", + ] + ), + tuple([script_name, 21, "main"]), + ) + ], + "Task-2": [ + ( + tuple([script_name, 11, "deep"]), + tuple([script_name, 15, "c1"]), + tuple( + [ + staggered.__file__, + ANY, + "staggered_race..run_one_coro", + ] + ), + ) + ], + }, + ) + + # Check awaited_by coroutine stacks + id_to_task = self._get_task_id_map(stack_trace) + awaited_by_coroutine_stacks = { + task.task_name: sorted( ( + id_to_task[coro.task_name].task_name, tuple( - [staggered.__file__, ANY, "staggered_race"] + tuple(frame) for frame in coro.call_stack ), - tuple([script_name, 21, "main"]), - ), + ) + for coro in task.awaited_by ) - ], - }, - ) + for task in stack_trace[0].awaited_by + } + self.assertEqual( + awaited_by_coroutine_stacks, + { + "Task-1": [], + "Task-2": [ + ( + "Task-1", + ( + tuple( + [ + staggered.__file__, + ANY, + "staggered_race", + ] + ), + tuple([script_name, 21, "main"]), + ), + ) + ], + }, + ) + finally: + _cleanup_sockets(client_socket, server_socket) @skip_if_not_supported @unittest.skipIf( @@ -818,6 +1126,10 @@ async def main(): "Test only runs on Linux with process_vm_readv support", ) def test_async_global_awaited_by(self): + # Reduced from 1000 to 100 to avoid file descriptor exhaustion + # when running tests in parallel (e.g., -j 20) + NUM_TASKS = 100 + port = find_unused_port() script = textwrap.dedent( f"""\ @@ -833,7 +1145,6 @@ def test_async_global_awaited_by(self): PORT = socket_helper.find_unused_port() connections = 0 - # Connect to the test process sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect(('localhost', {port})) @@ -856,23 +1167,16 @@ async def echo_client(message): assert message == data.decode() writer.close() await writer.wait_closed() - # Signal we are ready to sleep sock.sendall(b"ready") await asyncio.sleep(SHORT_TIMEOUT) async def echo_client_spam(server): async with asyncio.TaskGroup() as tg: - while connections < 1000: + while connections < {NUM_TASKS}: msg = list(ascii_lowercase + digits) random.shuffle(msg) tg.create_task(echo_client("".join(msg))) await asyncio.sleep(0) - # at least a 1000 tasks created. Each task will signal - # when is ready to avoid the race caused by the fact that - # tasks are waited on tg.__exit__ and we cannot signal when - # that happens otherwise - # at this point all client tasks completed without assertion errors - # let's wrap up the test server.close() await server.wait_closed() @@ -887,231 +1191,216 @@ async def main(): asyncio.run(main()) """ ) - stack_trace = None + with os_helper.temp_dir() as work_dir: script_dir = os.path.join(work_dir, "script_pkg") os.mkdir(script_dir) - # Create a socket server to communicate with the target process - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server_socket.bind(("localhost", port)) - server_socket.settimeout(SHORT_TIMEOUT) - server_socket.listen(1) + + server_socket = _create_server_socket(port) script_name = _make_test_script(script_dir, "script", script) client_socket = None + try: - p = subprocess.Popen([sys.executable, script_name]) - client_socket, _ = server_socket.accept() - server_socket.close() - for _ in range(1000): - expected_response = b"ready" - response = client_socket.recv(len(expected_response)) - self.assertEqual(response, expected_response) - for _ in busy_retry(SHORT_TIMEOUT): + with _managed_subprocess([sys.executable, script_name]) as p: + client_socket, _ = server_socket.accept() + server_socket.close() + server_socket = None + + # Wait for NUM_TASKS "ready" signals + try: + _wait_for_n_signals(client_socket, b"ready", NUM_TASKS) + except RuntimeError as e: + self.fail(str(e)) + try: all_awaited_by = get_all_awaited_by(p.pid) - except RuntimeError as re: - # This call reads a linked list in another process with - # no synchronization. That occasionally leads to invalid - # reads. Here we avoid making the test flaky. - msg = str(re) - if msg.startswith("Task list appears corrupted"): - continue - elif msg.startswith( - "Invalid linked list structure reading remote memory" - ): - continue - elif msg.startswith("Unknown error reading memory"): - continue - elif msg.startswith("Unhandled frame owner"): - continue - raise # Unrecognized exception, safest not to ignore it - else: - break - # expected: a list of two elements: 1 thread, 1 interp - self.assertEqual(len(all_awaited_by), 2) - # expected: a tuple with the thread ID and the awaited_by list - self.assertEqual(len(all_awaited_by[0]), 2) - # expected: no tasks in the fallback per-interp task list - self.assertEqual(all_awaited_by[1], (0, [])) - entries = all_awaited_by[0][1] - # expected: at least 1000 pending tasks - self.assertGreaterEqual(len(entries), 1000) - # the first three tasks stem from the code structure - main_stack = [ - FrameInfo([taskgroups.__file__, ANY, "TaskGroup._aexit"]), - FrameInfo( - [taskgroups.__file__, ANY, "TaskGroup.__aexit__"] - ), - FrameInfo([script_name, 60, "main"]), - ] - self.assertIn( - TaskInfo( - [ANY, "Task-1", [CoroInfo([main_stack, ANY])], []] - ), - entries, - ) - self.assertIn( - TaskInfo( - [ - ANY, - "server task", + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" + ) + + # Expected: a list of two elements: 1 thread, 1 interp + self.assertEqual(len(all_awaited_by), 2) + # Expected: a tuple with the thread ID and the awaited_by list + self.assertEqual(len(all_awaited_by[0]), 2) + # Expected: no tasks in the fallback per-interp task list + self.assertEqual(all_awaited_by[1], (0, [])) + + entries = all_awaited_by[0][1] + # Expected: at least NUM_TASKS pending tasks + self.assertGreaterEqual(len(entries), NUM_TASKS) + + # Check the main task structure + main_stack = [ + FrameInfo( + [taskgroups.__file__, ANY, "TaskGroup._aexit"] + ), + FrameInfo( + [taskgroups.__file__, ANY, "TaskGroup.__aexit__"] + ), + FrameInfo([script_name, 52, "main"]), + ] + self.assertIn( + TaskInfo( + [ANY, "Task-1", [CoroInfo([main_stack, ANY])], []] + ), + entries, + ) + self.assertIn( + TaskInfo( [ - CoroInfo( - [ + ANY, + "server task", + [ + CoroInfo( [ - FrameInfo( - [ - base_events.__file__, - ANY, - "Server.serve_forever", - ] - ) - ], - ANY, - ] - ) - ], - [ - CoroInfo( - [ + [ + FrameInfo( + [ + base_events.__file__, + ANY, + "Server.serve_forever", + ] + ) + ], + ANY, + ] + ) + ], + [ + CoroInfo( [ - FrameInfo( - [ - taskgroups.__file__, - ANY, - "TaskGroup._aexit", - ] - ), - FrameInfo( - [ - taskgroups.__file__, - ANY, - "TaskGroup.__aexit__", - ] - ), - FrameInfo( - [script_name, ANY, "main"] - ), - ], - ANY, - ] - ) - ], - ] - ), - entries, - ) - self.assertIn( - TaskInfo( - [ - ANY, - "Task-4", + [ + FrameInfo( + [ + taskgroups.__file__, + ANY, + "TaskGroup._aexit", + ] + ), + FrameInfo( + [ + taskgroups.__file__, + ANY, + "TaskGroup.__aexit__", + ] + ), + FrameInfo( + [script_name, ANY, "main"] + ), + ], + ANY, + ] + ) + ], + ] + ), + entries, + ) + self.assertIn( + TaskInfo( [ - CoroInfo( - [ + ANY, + "Task-4", + [ + CoroInfo( [ - FrameInfo( - [tasks.__file__, ANY, "sleep"] - ), - FrameInfo( - [ - script_name, - 38, - "echo_client", - ] - ), - ], - ANY, - ] - ) - ], - [ - CoroInfo( - [ + [ + FrameInfo( + [ + tasks.__file__, + ANY, + "sleep", + ] + ), + FrameInfo( + [ + script_name, + 36, + "echo_client", + ] + ), + ], + ANY, + ] + ) + ], + [ + CoroInfo( [ - FrameInfo( - [ - taskgroups.__file__, - ANY, - "TaskGroup._aexit", - ] - ), - FrameInfo( - [ - taskgroups.__file__, - ANY, - "TaskGroup.__aexit__", - ] - ), - FrameInfo( - [ - script_name, - 41, - "echo_client_spam", - ] - ), - ], - ANY, - ] - ) - ], - ] - ), - entries, - ) + [ + FrameInfo( + [ + taskgroups.__file__, + ANY, + "TaskGroup._aexit", + ] + ), + FrameInfo( + [ + taskgroups.__file__, + ANY, + "TaskGroup.__aexit__", + ] + ), + FrameInfo( + [ + script_name, + 39, + "echo_client_spam", + ] + ), + ], + ANY, + ] + ) + ], + ] + ), + entries, + ) - expected_awaited_by = [ - CoroInfo( - [ + expected_awaited_by = [ + CoroInfo( [ - FrameInfo( - [ - taskgroups.__file__, - ANY, - "TaskGroup._aexit", - ] - ), - FrameInfo( - [ - taskgroups.__file__, - ANY, - "TaskGroup.__aexit__", - ] - ), - FrameInfo( - [script_name, 41, "echo_client_spam"] - ), - ], - ANY, - ] - ) - ] - tasks_with_awaited = [ - task - for task in entries - if task.awaited_by == expected_awaited_by - ] - self.assertGreaterEqual(len(tasks_with_awaited), 1000) - - # the final task will have some random number, but it should for - # sure be one of the echo client spam horde (In windows this is not true - # for some reason) - if sys.platform != "win32": - self.assertEqual( - tasks_with_awaited[-1].awaited_by, - entries[-1].awaited_by, - ) - except PermissionError: - self.skipTest( - "Insufficient permissions to read the stack trace" - ) + [ + FrameInfo( + [ + taskgroups.__file__, + ANY, + "TaskGroup._aexit", + ] + ), + FrameInfo( + [ + taskgroups.__file__, + ANY, + "TaskGroup.__aexit__", + ] + ), + FrameInfo( + [script_name, 39, "echo_client_spam"] + ), + ], + ANY, + ] + ) + ] + tasks_with_awaited = [ + task + for task in entries + if task.awaited_by == expected_awaited_by + ] + self.assertGreaterEqual(len(tasks_with_awaited), NUM_TASKS) + + # Final task should be from echo client spam (not on Windows) + if sys.platform != "win32": + self.assertEqual( + tasks_with_awaited[-1].awaited_by, + entries[-1].awaited_by, + ) finally: - if client_socket is not None: - client_socket.close() - p.kill() - p.terminate() - p.wait(timeout=SHORT_TIMEOUT) + _cleanup_sockets(client_socket, server_socket) @skip_if_not_supported @unittest.skipIf( @@ -1120,25 +1409,24 @@ async def main(): ) def test_self_trace(self): stack_trace = get_stack_trace(os.getpid()) - # Is possible that there are more threads, so we check that the - # expected stack traces are in the result (looking at you Windows!) - this_tread_stack = None - # New format: [InterpreterInfo(interpreter_id, [ThreadInfo(...)])] + + this_thread_stack = None for interpreter_info in stack_trace: for thread_info in interpreter_info.threads: if thread_info.thread_id == threading.get_native_id(): - this_tread_stack = thread_info.frame_info + this_thread_stack = thread_info.frame_info break - if this_tread_stack: + if this_thread_stack: break - self.assertIsNotNone(this_tread_stack) + + self.assertIsNotNone(this_thread_stack) self.assertEqual( - this_tread_stack[:2], + this_thread_stack[:2], [ FrameInfo( [ __file__, - get_stack_trace.__code__.co_firstlineno + 2, + get_stack_trace.__code__.co_firstlineno + 4, "get_stack_trace", ] ), @@ -1159,12 +1447,11 @@ def test_self_trace(self): ) @requires_subinterpreters def test_subinterpreter_stack_trace(self): - # Test that subinterpreters are correctly handled port = find_unused_port() - # Calculate subinterpreter code separately and pickle it to avoid f-string issues import pickle - subinterp_code = textwrap.dedent(f''' + + subinterp_code = textwrap.dedent(f""" import socket import time @@ -1177,9 +1464,8 @@ def nested_func(): nested_func() sub_worker() - ''').strip() + """).strip() - # Pickle the subinterpreter code pickled_code = pickle.dumps(subinterp_code) script = textwrap.dedent( @@ -1190,33 +1476,26 @@ def nested_func(): import socket import threading - # Connect to the test process sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect(('localhost', {port})) def main_worker(): - # Function running in main interpreter sock.sendall(b"ready:main\\n") time.sleep(10_000) def run_subinterp(): - # Create and run subinterpreter subinterp = interpreters.create() - import pickle pickled_code = {pickled_code!r} subinterp_code = pickle.loads(pickled_code) subinterp.exec(subinterp_code) - # Start subinterpreter in thread sub_thread = threading.Thread(target=run_subinterp) sub_thread.start() - # Start main thread work main_thread = threading.Thread(target=main_worker) main_thread.start() - # Keep main thread alive main_thread.join() sub_thread.join() """ @@ -1226,85 +1505,74 @@ def run_subinterp(): script_dir = os.path.join(work_dir, "script_pkg") os.mkdir(script_dir) - # Create a socket server to communicate with the target process - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server_socket.bind(("localhost", port)) - server_socket.settimeout(SHORT_TIMEOUT) - server_socket.listen(1) - + server_socket = _create_server_socket(port) script_name = _make_test_script(script_dir, "script", script) client_sockets = [] - try: - p = subprocess.Popen([sys.executable, script_name]) - # Accept connections from both main and subinterpreter - responses = set() - while len(responses) < 2: # Wait for both "ready:main" and "ready:sub" - try: - client_socket, _ = server_socket.accept() - client_sockets.append(client_socket) + try: + with _managed_subprocess([sys.executable, script_name]) as p: + # Accept connections from both main and subinterpreter + responses = set() + while len(responses) < 2: + try: + client_socket, _ = server_socket.accept() + client_sockets.append(client_socket) + response = client_socket.recv(1024) + if b"ready:main" in response: + responses.add("main") + if b"ready:sub" in response: + responses.add("sub") + except socket.timeout: + break - # Read the response from this connection - response = client_socket.recv(1024) - if b"ready:main" in response: - responses.add("main") - if b"ready:sub" in response: - responses.add("sub") - except socket.timeout: - break + server_socket.close() + server_socket = None - server_socket.close() - stack_trace = get_stack_trace(p.pid) - except PermissionError: - self.skipTest( - "Insufficient permissions to read the stack trace" - ) - finally: - for client_socket in client_sockets: - if client_socket is not None: - client_socket.close() - p.kill() - p.terminate() - p.wait(timeout=SHORT_TIMEOUT) + try: + stack_trace = get_stack_trace(p.pid) + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" + ) - # Verify we have multiple interpreters - self.assertGreaterEqual(len(stack_trace), 1, "Should have at least one interpreter") + # Verify we have at least one interpreter + self.assertGreaterEqual(len(stack_trace), 1) - # Look for main interpreter (ID 0) and subinterpreter (ID > 0) - main_interp = None - sub_interp = None + # Look for main interpreter (ID 0) and subinterpreter (ID > 0) + main_interp = None + sub_interp = None + for interpreter_info in stack_trace: + if interpreter_info.interpreter_id == 0: + main_interp = interpreter_info + elif interpreter_info.interpreter_id > 0: + sub_interp = interpreter_info - for interpreter_info in stack_trace: - if interpreter_info.interpreter_id == 0: - main_interp = interpreter_info - elif interpreter_info.interpreter_id > 0: - sub_interp = interpreter_info + self.assertIsNotNone( + main_interp, "Main interpreter should be present" + ) - self.assertIsNotNone(main_interp, "Main interpreter should be present") + # Check main interpreter has expected stack trace + main_found = self._find_frame_in_trace( + [main_interp], lambda f: f.funcname == "main_worker" + ) + self.assertIsNotNone( + main_found, + "Main interpreter should have main_worker in stack", + ) - # Check main interpreter has expected stack trace - main_found = False - for thread_info in main_interp.threads: - for frame in thread_info.frame_info: - if frame.funcname == "main_worker": - main_found = True - break - if main_found: - break - self.assertTrue(main_found, "Main interpreter should have main_worker in stack") - - # If subinterpreter is present, check its stack trace - if sub_interp: - sub_found = False - for thread_info in sub_interp.threads: - for frame in thread_info.frame_info: - if frame.funcname in ("sub_worker", "nested_func"): - sub_found = True - break - if sub_found: - break - self.assertTrue(sub_found, "Subinterpreter should have sub_worker or nested_func in stack") + # If subinterpreter is present, check its stack trace + if sub_interp: + sub_found = self._find_frame_in_trace( + [sub_interp], + lambda f: f.funcname + in ("sub_worker", "nested_func"), + ) + self.assertIsNotNone( + sub_found, + "Subinterpreter should have sub_worker or nested_func in stack", + ) + finally: + _cleanup_sockets(*client_sockets, server_socket) @skip_if_not_supported @unittest.skipIf( @@ -1313,14 +1581,11 @@ def run_subinterp(): ) @requires_subinterpreters def test_multiple_subinterpreters_with_threads(self): - # Test multiple subinterpreters, each with multiple threads port = find_unused_port() - # Calculate subinterpreter codes separately and pickle them import pickle - # Code for first subinterpreter with 2 threads - subinterp1_code = textwrap.dedent(f''' + subinterp1_code = textwrap.dedent(f""" import socket import time import threading @@ -1347,10 +1612,9 @@ def nested_func(): t2.start() t1.join() t2.join() - ''').strip() + """).strip() - # Code for second subinterpreter with 2 threads - subinterp2_code = textwrap.dedent(f''' + subinterp2_code = textwrap.dedent(f""" import socket import time import threading @@ -1377,9 +1641,8 @@ def nested_func(): t2.start() t1.join() t2.join() - ''').strip() + """).strip() - # Pickle the subinterpreter codes pickled_code1 = pickle.dumps(subinterp1_code) pickled_code2 = pickle.dumps(subinterp2_code) @@ -1391,44 +1654,35 @@ def nested_func(): import socket import threading - # Connect to the test process sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect(('localhost', {port})) def main_worker(): - # Function running in main interpreter sock.sendall(b"ready:main\\n") time.sleep(10_000) def run_subinterp1(): - # Create and run first subinterpreter subinterp = interpreters.create() - import pickle pickled_code = {pickled_code1!r} subinterp_code = pickle.loads(pickled_code) subinterp.exec(subinterp_code) def run_subinterp2(): - # Create and run second subinterpreter subinterp = interpreters.create() - import pickle pickled_code = {pickled_code2!r} subinterp_code = pickle.loads(pickled_code) subinterp.exec(subinterp_code) - # Start subinterpreters in threads sub1_thread = threading.Thread(target=run_subinterp1) sub2_thread = threading.Thread(target=run_subinterp2) sub1_thread.start() sub2_thread.start() - # Start main thread work main_thread = threading.Thread(target=main_worker) main_thread.start() - # Keep main thread alive main_thread.join() sub1_thread.join() sub2_thread.join() @@ -1439,72 +1693,80 @@ def run_subinterp2(): script_dir = os.path.join(work_dir, "script_pkg") os.mkdir(script_dir) - # Create a socket server to communicate with the target process - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server_socket.bind(("localhost", port)) - server_socket.settimeout(SHORT_TIMEOUT) - server_socket.listen(5) # Allow multiple connections - + server_socket = _create_server_socket(port, backlog=5) script_name = _make_test_script(script_dir, "script", script) client_sockets = [] + try: - p = subprocess.Popen([sys.executable, script_name]) + with _managed_subprocess([sys.executable, script_name]) as p: + # Accept connections from main and all subinterpreter threads + expected_responses = { + "ready:main", + "ready:sub1-t1", + "ready:sub1-t2", + "ready:sub2-t1", + "ready:sub2-t2", + } + responses = set() + + while len(responses) < 5: + try: + client_socket, _ = server_socket.accept() + client_sockets.append(client_socket) + response = client_socket.recv(1024) + response_str = response.decode().strip() + if response_str in expected_responses: + responses.add(response_str) + except socket.timeout: + break - # Accept connections from main and all subinterpreter threads - expected_responses = {"ready:main", "ready:sub1-t1", "ready:sub1-t2", "ready:sub2-t1", "ready:sub2-t2"} - responses = set() + server_socket.close() + server_socket = None - while len(responses) < 5: # Wait for all 5 ready signals try: - client_socket, _ = server_socket.accept() - client_sockets.append(client_socket) - - # Read the response from this connection - response = client_socket.recv(1024) - response_str = response.decode().strip() - if response_str in expected_responses: - responses.add(response_str) - except socket.timeout: - break - - server_socket.close() - stack_trace = get_stack_trace(p.pid) - except PermissionError: - self.skipTest( - "Insufficient permissions to read the stack trace" - ) - finally: - for client_socket in client_sockets: - if client_socket is not None: - client_socket.close() - p.kill() - p.terminate() - p.wait(timeout=SHORT_TIMEOUT) - - # Verify we have multiple interpreters - self.assertGreaterEqual(len(stack_trace), 2, "Should have at least two interpreters") - - # Count interpreters by ID - interpreter_ids = {interp.interpreter_id for interp in stack_trace} - self.assertIn(0, interpreter_ids, "Main interpreter should be present") - self.assertGreaterEqual(len(interpreter_ids), 3, "Should have main + at least 2 subinterpreters") + stack_trace = get_stack_trace(p.pid) + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" + ) - # Count total threads across all interpreters - total_threads = sum(len(interp.threads) for interp in stack_trace) - self.assertGreaterEqual(total_threads, 5, "Should have at least 5 threads total") + # Verify we have multiple interpreters + self.assertGreaterEqual(len(stack_trace), 2) + + # Count interpreters by ID + interpreter_ids = { + interp.interpreter_id for interp in stack_trace + } + self.assertIn( + 0, + interpreter_ids, + "Main interpreter should be present", + ) + self.assertGreaterEqual(len(interpreter_ids), 3) - # Look for expected function names in stack traces - all_funcnames = set() - for interpreter_info in stack_trace: - for thread_info in interpreter_info.threads: - for frame in thread_info.frame_info: - all_funcnames.add(frame.funcname) + # Count total threads + total_threads = sum( + len(interp.threads) for interp in stack_trace + ) + self.assertGreaterEqual(total_threads, 5) - # Should find functions from different interpreters and threads - expected_funcs = {"main_worker", "worker1", "worker2", "nested_func"} - found_funcs = expected_funcs.intersection(all_funcnames) - self.assertGreater(len(found_funcs), 0, f"Should find some expected functions, got: {all_funcnames}") + # Look for expected function names + all_funcnames = set() + for interpreter_info in stack_trace: + for thread_info in interpreter_info.threads: + for frame in thread_info.frame_info: + all_funcnames.add(frame.funcname) + + expected_funcs = { + "main_worker", + "worker1", + "worker2", + "nested_func", + } + found_funcs = expected_funcs.intersection(all_funcnames) + self.assertGreater(len(found_funcs), 0) + finally: + _cleanup_sockets(*client_sockets, server_socket) @skip_if_not_supported @unittest.skipIf( @@ -1513,54 +1775,41 @@ def run_subinterp2(): ) @requires_gil_enabled("Free threaded builds don't have an 'active thread'") def test_only_active_thread(self): - # Test that only_active_thread parameter works correctly port = find_unused_port() script = textwrap.dedent( f"""\ import time, sys, socket, threading - # Connect to the test process sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect(('localhost', {port})) def worker_thread(name, barrier, ready_event): - barrier.wait() # Synchronize thread start - ready_event.wait() # Wait for main thread signal - # Sleep to keep thread alive + barrier.wait() + ready_event.wait() time.sleep(10_000) def main_work(): - # Do busy work to hold the GIL sock.sendall(b"working\\n") count = 0 while count < 100000000: count += 1 if count % 10000000 == 0: - pass # Keep main thread busy + pass sock.sendall(b"done\\n") - # Create synchronization primitives num_threads = 3 - barrier = threading.Barrier(num_threads + 1) # +1 for main thread + barrier = threading.Barrier(num_threads + 1) ready_event = threading.Event() - # Start worker threads threads = [] for i in range(num_threads): t = threading.Thread(target=worker_thread, args=(f"Worker-{{i}}", barrier, ready_event)) t.start() threads.append(t) - # Wait for all threads to be ready barrier.wait() - - # Signal ready to parent process sock.sendall(b"ready\\n") - - # Signal threads to start waiting ready_event.set() - - # Now do busy work to hold the GIL main_work() """ ) @@ -1569,104 +1818,76 @@ def main_work(): script_dir = os.path.join(work_dir, "script_pkg") os.mkdir(script_dir) - # Create a socket server to communicate with the target process - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server_socket.bind(("localhost", port)) - server_socket.settimeout(SHORT_TIMEOUT) - server_socket.listen(1) - + server_socket = _create_server_socket(port) script_name = _make_test_script(script_dir, "script", script) client_socket = None - try: - p = subprocess.Popen([sys.executable, script_name]) - client_socket, _ = server_socket.accept() - server_socket.close() - - # Wait for ready signal - response = b"" - while b"ready" not in response: - response += client_socket.recv(1024) - - # Wait for the main thread to start its busy work - while b"working" not in response: - response += client_socket.recv(1024) - - # Get stack trace with all threads - unwinder_all = RemoteUnwinder(p.pid, all_threads=True) - for _ in range(10): - # Wait for the main thread to start its busy work - all_traces = unwinder_all.get_stack_trace() - found = False - # New format: [InterpreterInfo(interpreter_id, [ThreadInfo(...)])] - for interpreter_info in all_traces: - for thread_info in interpreter_info.threads: - if not thread_info.frame_info: - continue - current_frame = thread_info.frame_info[0] - if ( - current_frame.funcname == "main_work" - and current_frame.lineno > 15 - ): - found = True - break - if found: - break - if found: - break - # Give a bit of time to take the next sample - time.sleep(0.1) - else: - self.fail( - "Main thread did not start its busy work on time" - ) + try: + with _managed_subprocess([sys.executable, script_name]) as p: + client_socket, _ = server_socket.accept() + server_socket.close() + server_socket = None - # Get stack trace with only GIL holder - unwinder_gil = RemoteUnwinder(p.pid, only_active_thread=True) - gil_traces = unwinder_gil.get_stack_trace() + # Wait for ready and working signals + _wait_for_signal(client_socket, [b"ready", b"working"]) - except PermissionError: - self.skipTest( - "Insufficient permissions to read the stack trace" - ) - finally: - if client_socket is not None: - client_socket.close() - p.kill() - p.terminate() - p.wait(timeout=SHORT_TIMEOUT) + try: + # Get stack trace with all threads + unwinder_all = RemoteUnwinder(p.pid, all_threads=True) + for _ in range(MAX_TRIES): + all_traces = unwinder_all.get_stack_trace() + found = self._find_frame_in_trace( + all_traces, + lambda f: f.funcname == "main_work" + and f.lineno > 12, + ) + if found: + break + time.sleep(0.1) + else: + self.fail( + "Main thread did not start its busy work on time" + ) - # Count total threads across all interpreters in all_traces - total_threads = sum(len(interpreter_info.threads) for interpreter_info in all_traces) - self.assertGreater( - total_threads, 1, "Should have multiple threads" - ) + # Get stack trace with only GIL holder + unwinder_gil = RemoteUnwinder( + p.pid, only_active_thread=True + ) + gil_traces = unwinder_gil.get_stack_trace() + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" + ) - # Count total threads across all interpreters in gil_traces - total_gil_threads = sum(len(interpreter_info.threads) for interpreter_info in gil_traces) - self.assertEqual( - total_gil_threads, 1, "Should have exactly one GIL holder" - ) + # Count threads + total_threads = sum( + len(interp.threads) for interp in all_traces + ) + self.assertGreater(total_threads, 1) - # Get the GIL holder thread ID - gil_thread_id = None - for interpreter_info in gil_traces: - if interpreter_info.threads: - gil_thread_id = interpreter_info.threads[0].thread_id - break + total_gil_threads = sum( + len(interp.threads) for interp in gil_traces + ) + self.assertEqual(total_gil_threads, 1) + + # Get the GIL holder thread ID + gil_thread_id = None + for interpreter_info in gil_traces: + if interpreter_info.threads: + gil_thread_id = interpreter_info.threads[ + 0 + ].thread_id + break - # Get all thread IDs from all_traces - all_thread_ids = [] - for interpreter_info in all_traces: - for thread_info in interpreter_info.threads: - all_thread_ids.append(thread_info.thread_id) + # Get all thread IDs + all_thread_ids = [] + for interpreter_info in all_traces: + for thread_info in interpreter_info.threads: + all_thread_ids.append(thread_info.thread_id) - self.assertIn( - gil_thread_id, - all_thread_ids, - "GIL holder should be among all threads", - ) + self.assertIn(gil_thread_id, all_thread_ids) + finally: + _cleanup_sockets(client_socket, server_socket) class TestUnsupportedPlatformHandling(unittest.TestCase): @@ -1674,23 +1895,28 @@ class TestUnsupportedPlatformHandling(unittest.TestCase): sys.platform in ("linux", "darwin", "win32"), "Test only runs on unsupported platforms (not Linux, macOS, or Windows)", ) - @unittest.skipIf(sys.platform == "android", "Android raises Linux-specific exception") + @unittest.skipIf( + sys.platform == "android", "Android raises Linux-specific exception" + ) def test_unsupported_platform_error(self): with self.assertRaises(RuntimeError) as cm: RemoteUnwinder(os.getpid()) self.assertIn( "Reading the PyRuntime section is not supported on this platform", - str(cm.exception) + str(cm.exception), ) -class TestDetectionOfThreadStatus(unittest.TestCase): - @unittest.skipIf( - sys.platform not in ("linux", "darwin", "win32"), - "Test only runs on unsupported platforms (not Linux, macOS, or Windows)", - ) - @unittest.skipIf(sys.platform == "android", "Android raises Linux-specific exception") - def test_thread_status_detection(self): + +class TestDetectionOfThreadStatus(RemoteInspectionTestBase): + def _run_thread_status_test(self, mode, check_condition): + """ + Common pattern for thread status detection tests. + + Args: + mode: Profiling mode (PROFILING_MODE_CPU, PROFILING_MODE_GIL, etc.) + check_condition: Function(statuses, sleeper_tid, busy_tid) -> bool + """ port = find_unused_port() script = textwrap.dedent( f"""\ @@ -1723,203 +1949,146 @@ def busy(): sock.close() """ ) + with os_helper.temp_dir() as work_dir: script_dir = os.path.join(work_dir, "script_pkg") os.mkdir(script_dir) - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server_socket.bind(("localhost", port)) - server_socket.settimeout(SHORT_TIMEOUT) - server_socket.listen(1) - script_name = _make_test_script(script_dir, "thread_status_script", script) + server_socket = _create_server_socket(port) + script_name = _make_test_script( + script_dir, "thread_status_script", script + ) client_socket = None + try: - p = subprocess.Popen([sys.executable, script_name]) - client_socket, _ = server_socket.accept() - server_socket.close() - response = b"" - sleeper_tid = None - busy_tid = None - while True: - chunk = client_socket.recv(1024) - response += chunk - if b"ready:main" in response and b"ready:sleeper" in response and b"ready:busy" in response: - # Parse TIDs from the response - for line in response.split(b"\n"): - if line.startswith(b"ready:sleeper:"): - try: - sleeper_tid = int(line.split(b":")[-1]) - except Exception: - pass - elif line.startswith(b"ready:busy:"): - try: - busy_tid = int(line.split(b":")[-1]) - except Exception: - pass - break + with _managed_subprocess([sys.executable, script_name]) as p: + client_socket, _ = server_socket.accept() + server_socket.close() + server_socket = None - attempts = 10 - statuses = {} - try: - unwinder = RemoteUnwinder(p.pid, all_threads=True, mode=PROFILING_MODE_CPU, - skip_non_matching_threads=False) - for _ in range(attempts): - traces = unwinder.get_stack_trace() - # Find threads and their statuses - statuses = {} - for interpreter_info in traces: - for thread_info in interpreter_info.threads: - statuses[thread_info.thread_id] = thread_info.status - - # Check if sleeper thread is off CPU and busy thread is on CPU - # In the new flags system: - # - sleeper should NOT have ON_CPU flag (off CPU) - # - busy should have ON_CPU flag - if (sleeper_tid in statuses and - busy_tid in statuses and - not (statuses[sleeper_tid] & THREAD_STATUS_ON_CPU) and - (statuses[busy_tid] & THREAD_STATUS_ON_CPU)): - break - time.sleep(0.5) # Give a bit of time to let threads settle - except PermissionError: - self.skipTest( - "Insufficient permissions to read the stack trace" + # Wait for all ready signals and parse TIDs + response = _wait_for_signal( + client_socket, + [b"ready:main", b"ready:sleeper", b"ready:busy"], + ) + + sleeper_tid = None + busy_tid = None + for line in response.split(b"\n"): + if line.startswith(b"ready:sleeper:"): + try: + sleeper_tid = int(line.split(b":")[-1]) + except (ValueError, IndexError): + pass + elif line.startswith(b"ready:busy:"): + try: + busy_tid = int(line.split(b":")[-1]) + except (ValueError, IndexError): + pass + + self.assertIsNotNone( + sleeper_tid, "Sleeper thread id not received" ) + self.assertIsNotNone( + busy_tid, "Busy thread id not received" + ) + + # Sample until we see expected thread states + statuses = {} + try: + unwinder = RemoteUnwinder( + p.pid, + all_threads=True, + mode=mode, + skip_non_matching_threads=False, + ) + for _ in range(MAX_TRIES): + traces = unwinder.get_stack_trace() + statuses = self._get_thread_statuses(traces) - self.assertIsNotNone(sleeper_tid, "Sleeper thread id not received") - self.assertIsNotNone(busy_tid, "Busy thread id not received") - self.assertIn(sleeper_tid, statuses, "Sleeper tid not found in sampled threads") - self.assertIn(busy_tid, statuses, "Busy tid not found in sampled threads") - self.assertFalse(statuses[sleeper_tid] & THREAD_STATUS_ON_CPU, "Sleeper thread should be off CPU") - self.assertTrue(statuses[busy_tid] & THREAD_STATUS_ON_CPU, "Busy thread should be on CPU") + if check_condition( + statuses, sleeper_tid, busy_tid + ): + break + time.sleep(0.5) + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" + ) + return statuses, sleeper_tid, busy_tid finally: - if client_socket is not None: - client_socket.close() - p.terminate() - p.wait(timeout=SHORT_TIMEOUT) + _cleanup_sockets(client_socket, server_socket) @unittest.skipIf( sys.platform not in ("linux", "darwin", "win32"), - "Test only runs on unsupported platforms (not Linux, macOS, or Windows)", + "Test only runs on supported platforms (Linux, macOS, or Windows)", ) - @unittest.skipIf(sys.platform == "android", "Android raises Linux-specific exception") - def test_thread_status_gil_detection(self): - port = find_unused_port() - script = textwrap.dedent( - f"""\ - import time, sys, socket, threading - import os - - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect(('localhost', {port})) - - def sleeper(): - tid = threading.get_native_id() - sock.sendall(f'ready:sleeper:{{tid}}\\n'.encode()) - time.sleep(10000) - - def busy(): - tid = threading.get_native_id() - sock.sendall(f'ready:busy:{{tid}}\\n'.encode()) - x = 0 - while True: - x = x + 1 - time.sleep(0.5) + @unittest.skipIf( + sys.platform == "android", "Android raises Linux-specific exception" + ) + def test_thread_status_detection(self): + def check_cpu_status(statuses, sleeper_tid, busy_tid): + return ( + sleeper_tid in statuses + and busy_tid in statuses + and not (statuses[sleeper_tid] & THREAD_STATUS_ON_CPU) + and (statuses[busy_tid] & THREAD_STATUS_ON_CPU) + ) - t1 = threading.Thread(target=sleeper) - t2 = threading.Thread(target=busy) - t1.start() - t2.start() - sock.sendall(b'ready:main\\n') - t1.join() - t2.join() - sock.close() - """ + statuses, sleeper_tid, busy_tid = self._run_thread_status_test( + PROFILING_MODE_CPU, check_cpu_status ) - with os_helper.temp_dir() as work_dir: - script_dir = os.path.join(work_dir, "script_pkg") - os.mkdir(script_dir) - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server_socket.bind(("localhost", port)) - server_socket.settimeout(SHORT_TIMEOUT) - server_socket.listen(1) - script_name = _make_test_script(script_dir, "thread_status_script", script) - client_socket = None - try: - p = subprocess.Popen([sys.executable, script_name]) - client_socket, _ = server_socket.accept() - server_socket.close() - response = b"" - sleeper_tid = None - busy_tid = None - while True: - chunk = client_socket.recv(1024) - response += chunk - if b"ready:main" in response and b"ready:sleeper" in response and b"ready:busy" in response: - # Parse TIDs from the response - for line in response.split(b"\n"): - if line.startswith(b"ready:sleeper:"): - try: - sleeper_tid = int(line.split(b":")[-1]) - except Exception: - pass - elif line.startswith(b"ready:busy:"): - try: - busy_tid = int(line.split(b":")[-1]) - except Exception: - pass - break + self.assertIn(sleeper_tid, statuses) + self.assertIn(busy_tid, statuses) + self.assertFalse( + statuses[sleeper_tid] & THREAD_STATUS_ON_CPU, + "Sleeper thread should be off CPU", + ) + self.assertTrue( + statuses[busy_tid] & THREAD_STATUS_ON_CPU, + "Busy thread should be on CPU", + ) - attempts = 10 - statuses = {} - try: - unwinder = RemoteUnwinder(p.pid, all_threads=True, mode=PROFILING_MODE_GIL, - skip_non_matching_threads=False) - for _ in range(attempts): - traces = unwinder.get_stack_trace() - # Find threads and their statuses - statuses = {} - for interpreter_info in traces: - for thread_info in interpreter_info.threads: - statuses[thread_info.thread_id] = thread_info.status - - # Check if sleeper thread doesn't have GIL and busy thread has GIL - # In the new flags system: - # - sleeper should NOT have HAS_GIL flag (waiting for GIL) - # - busy should have HAS_GIL flag - if (sleeper_tid in statuses and - busy_tid in statuses and - not (statuses[sleeper_tid] & THREAD_STATUS_HAS_GIL) and - (statuses[busy_tid] & THREAD_STATUS_HAS_GIL)): - break - time.sleep(0.5) # Give a bit of time to let threads settle - except PermissionError: - self.skipTest( - "Insufficient permissions to read the stack trace" - ) + @unittest.skipIf( + sys.platform not in ("linux", "darwin", "win32"), + "Test only runs on supported platforms (Linux, macOS, or Windows)", + ) + @unittest.skipIf( + sys.platform == "android", "Android raises Linux-specific exception" + ) + def test_thread_status_gil_detection(self): + def check_gil_status(statuses, sleeper_tid, busy_tid): + return ( + sleeper_tid in statuses + and busy_tid in statuses + and not (statuses[sleeper_tid] & THREAD_STATUS_HAS_GIL) + and (statuses[busy_tid] & THREAD_STATUS_HAS_GIL) + ) - self.assertIsNotNone(sleeper_tid, "Sleeper thread id not received") - self.assertIsNotNone(busy_tid, "Busy thread id not received") - self.assertIn(sleeper_tid, statuses, "Sleeper tid not found in sampled threads") - self.assertIn(busy_tid, statuses, "Busy tid not found in sampled threads") - self.assertFalse(statuses[sleeper_tid] & THREAD_STATUS_HAS_GIL, "Sleeper thread should not have GIL") - self.assertTrue(statuses[busy_tid] & THREAD_STATUS_HAS_GIL, "Busy thread should have GIL") + statuses, sleeper_tid, busy_tid = self._run_thread_status_test( + PROFILING_MODE_GIL, check_gil_status + ) - finally: - if client_socket is not None: - client_socket.close() - p.terminate() - p.wait(timeout=SHORT_TIMEOUT) + self.assertIn(sleeper_tid, statuses) + self.assertIn(busy_tid, statuses) + self.assertFalse( + statuses[sleeper_tid] & THREAD_STATUS_HAS_GIL, + "Sleeper thread should not have GIL", + ) + self.assertTrue( + statuses[busy_tid] & THREAD_STATUS_HAS_GIL, + "Busy thread should have GIL", + ) @unittest.skipIf( sys.platform not in ("linux", "darwin", "win32"), "Test only runs on supported platforms (Linux, macOS, or Windows)", ) - @unittest.skipIf(sys.platform == "android", "Android raises Linux-specific exception") + @unittest.skipIf( + sys.platform == "android", "Android raises Linux-specific exception" + ) def test_thread_status_all_mode_detection(self): port = find_unused_port() script = textwrap.dedent( @@ -1952,104 +2121,112 @@ def busy_thread(): with os_helper.temp_dir() as tmp_dir: script_file = make_script(tmp_dir, "script", script) - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server_socket.bind(("localhost", port)) - server_socket.listen(2) - server_socket.settimeout(SHORT_TIMEOUT) - - p = subprocess.Popen( - [sys.executable, script_file], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - + server_socket = _create_server_socket(port, backlog=2) client_sockets = [] - try: - sleeper_tid = None - busy_tid = None - # Receive thread IDs from the child process - for _ in range(2): - client_socket, _ = server_socket.accept() - client_sockets.append(client_socket) - line = client_socket.recv(1024) - if line: - if line.startswith(b"sleeper:"): - try: - sleeper_tid = int(line.split(b":")[-1]) - except Exception: - pass - elif line.startswith(b"busy:"): - try: - busy_tid = int(line.split(b":")[-1]) - except Exception: - pass + try: + with _managed_subprocess( + [sys.executable, script_file], + ) as p: + sleeper_tid = None + busy_tid = None + + # Receive thread IDs from the child process + for _ in range(2): + client_socket, _ = server_socket.accept() + client_sockets.append(client_socket) + line = client_socket.recv(1024) + if line: + if line.startswith(b"sleeper:"): + try: + sleeper_tid = int(line.split(b":")[-1]) + except (ValueError, IndexError): + pass + elif line.startswith(b"busy:"): + try: + busy_tid = int(line.split(b":")[-1]) + except (ValueError, IndexError): + pass - server_socket.close() + server_socket.close() + server_socket = None - attempts = 10 - statuses = {} - try: - unwinder = RemoteUnwinder(p.pid, all_threads=True, mode=PROFILING_MODE_ALL, - skip_non_matching_threads=False) - for _ in range(attempts): - traces = unwinder.get_stack_trace() - # Find threads and their statuses - statuses = {} - for interpreter_info in traces: - for thread_info in interpreter_info.threads: - statuses[thread_info.thread_id] = thread_info.status - - # Check ALL mode provides both GIL and CPU info - # - sleeper should NOT have ON_CPU and NOT have HAS_GIL - # - busy should have ON_CPU and have HAS_GIL - if (sleeper_tid in statuses and - busy_tid in statuses and - not (statuses[sleeper_tid] & THREAD_STATUS_ON_CPU) and - not (statuses[sleeper_tid] & THREAD_STATUS_HAS_GIL) and - (statuses[busy_tid] & THREAD_STATUS_ON_CPU) and - (statuses[busy_tid] & THREAD_STATUS_HAS_GIL)): - break - time.sleep(0.5) - except PermissionError: - self.skipTest( - "Insufficient permissions to read the stack trace" - ) + statuses = {} + try: + unwinder = RemoteUnwinder( + p.pid, + all_threads=True, + mode=PROFILING_MODE_ALL, + skip_non_matching_threads=False, + ) + for _ in range(MAX_TRIES): + traces = unwinder.get_stack_trace() + statuses = self._get_thread_statuses(traces) - self.assertIsNotNone(sleeper_tid, "Sleeper thread id not received") - self.assertIsNotNone(busy_tid, "Busy thread id not received") - self.assertIn(sleeper_tid, statuses, "Sleeper tid not found in sampled threads") - self.assertIn(busy_tid, statuses, "Busy tid not found in sampled threads") + # Check ALL mode provides both GIL and CPU info + if ( + sleeper_tid in statuses + and busy_tid in statuses + and not ( + statuses[sleeper_tid] + & THREAD_STATUS_ON_CPU + ) + and not ( + statuses[sleeper_tid] + & THREAD_STATUS_HAS_GIL + ) + and (statuses[busy_tid] & THREAD_STATUS_ON_CPU) + and ( + statuses[busy_tid] & THREAD_STATUS_HAS_GIL + ) + ): + break + time.sleep(0.5) + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" + ) - # Sleeper thread: off CPU, no GIL - self.assertFalse(statuses[sleeper_tid] & THREAD_STATUS_ON_CPU, "Sleeper should be off CPU") - self.assertFalse(statuses[sleeper_tid] & THREAD_STATUS_HAS_GIL, "Sleeper should not have GIL") + self.assertIsNotNone( + sleeper_tid, "Sleeper thread id not received" + ) + self.assertIsNotNone( + busy_tid, "Busy thread id not received" + ) + self.assertIn(sleeper_tid, statuses) + self.assertIn(busy_tid, statuses) - # Busy thread: on CPU, has GIL - self.assertTrue(statuses[busy_tid] & THREAD_STATUS_ON_CPU, "Busy should be on CPU") - self.assertTrue(statuses[busy_tid] & THREAD_STATUS_HAS_GIL, "Busy should have GIL") + # Sleeper: off CPU, no GIL + self.assertFalse( + statuses[sleeper_tid] & THREAD_STATUS_ON_CPU, + "Sleeper should be off CPU", + ) + self.assertFalse( + statuses[sleeper_tid] & THREAD_STATUS_HAS_GIL, + "Sleeper should not have GIL", + ) + # Busy: on CPU, has GIL + self.assertTrue( + statuses[busy_tid] & THREAD_STATUS_ON_CPU, + "Busy should be on CPU", + ) + self.assertTrue( + statuses[busy_tid] & THREAD_STATUS_HAS_GIL, + "Busy should have GIL", + ) finally: - for client_socket in client_sockets: - client_socket.close() - p.terminate() - p.wait(timeout=SHORT_TIMEOUT) - p.stdout.close() - p.stderr.close() + _cleanup_sockets(*client_sockets, server_socket) -class TestFrameCaching(unittest.TestCase): +class TestFrameCaching(RemoteInspectionTestBase): """Test that frame caching produces correct results. Uses socket-based synchronization for deterministic testing. All tests verify cache reuse via object identity checks (assertIs). """ - maxDiff = None - MAX_TRIES = 10 - - @contextlib.contextmanager + @contextmanager def _target_process(self, script_body): """Context manager for running a target process with socket sync.""" port = find_unused_port() @@ -2064,61 +2241,62 @@ def _target_process(self, script_body): script_dir = os.path.join(work_dir, "script_pkg") os.mkdir(script_dir) - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server_socket.bind(("localhost", port)) - server_socket.settimeout(SHORT_TIMEOUT) - server_socket.listen(1) - + server_socket = _create_server_socket(port) script_name = _make_test_script(script_dir, "script", script) client_socket = None - p = None + try: - p = subprocess.Popen([sys.executable, script_name]) - client_socket, _ = server_socket.accept() - server_socket.close() + with _managed_subprocess([sys.executable, script_name]) as p: + client_socket, _ = server_socket.accept() + server_socket.close() + server_socket = None - def make_unwinder(cache_frames=True): - return RemoteUnwinder(p.pid, all_threads=True, cache_frames=cache_frames) + def make_unwinder(cache_frames=True): + return RemoteUnwinder( + p.pid, all_threads=True, cache_frames=cache_frames + ) - yield p, client_socket, make_unwinder + yield p, client_socket, make_unwinder except PermissionError: - self.skipTest("Insufficient permissions to read the stack trace") + self.skipTest( + "Insufficient permissions to read the stack trace" + ) finally: - if client_socket: - client_socket.close() - if p: - p.kill() - p.terminate() - p.wait(timeout=SHORT_TIMEOUT) - - def _wait_for_signal(self, client_socket, signal): - """Block until signal received from target.""" - response = b"" - while signal not in response: - chunk = client_socket.recv(64) - if not chunk: - break - response += chunk - return response - - def _get_frames(self, unwinder, required_funcs): - """Sample and return frame_info list for thread containing required_funcs.""" - traces = unwinder.get_stack_trace() - for interp in traces: - for thread in interp.threads: - funcs = [f.funcname for f in thread.frame_info] - if required_funcs.issubset(set(funcs)): - return thread.frame_info + _cleanup_sockets(client_socket, server_socket) + + def _get_frames_with_retry(self, unwinder, required_funcs): + """Get frames containing required_funcs, with retry for transient errors.""" + for _ in range(MAX_TRIES): + try: + traces = unwinder.get_stack_trace() + for interp in traces: + for thread in interp.threads: + funcs = {f.funcname for f in thread.frame_info} + if required_funcs.issubset(funcs): + return thread.frame_info + except RuntimeError as e: + if _is_retriable_error(e): + pass + else: + raise + time.sleep(0.1) return None - def _sample_frames(self, client_socket, unwinder, wait_signal, send_ack, required_funcs, expected_frames=1): - """Wait for signal, sample frames, send ack. Returns frame_info list.""" - self._wait_for_signal(client_socket, wait_signal) - # Give at least MAX_TRIES tries for the process to arrive to a steady state - for _ in range(self.MAX_TRIES): - frames = self._get_frames(unwinder, required_funcs) + def _sample_frames( + self, + client_socket, + unwinder, + wait_signal, + send_ack, + required_funcs, + expected_frames=1, + ): + """Wait for signal, sample frames with retry until required funcs present, send ack.""" + _wait_for_signal(client_socket, wait_signal) + frames = None + for _ in range(MAX_TRIES): + frames = self._get_frames_with_retry(unwinder, required_funcs) if frames and len(frames) >= expected_frames: break time.sleep(0.1) @@ -2155,13 +2333,23 @@ def level1(): level1() """ - with self._target_process(script_body) as (p, client_socket, make_unwinder): + with self._target_process(script_body) as ( + p, + client_socket, + make_unwinder, + ): unwinder = make_unwinder(cache_frames=True) expected = {"level1", "level2", "level3"} - frames1 = self._sample_frames(client_socket, unwinder, b"sync1", b"ack", expected) - frames2 = self._sample_frames(client_socket, unwinder, b"sync2", b"ack", expected) - frames3 = self._sample_frames(client_socket, unwinder, b"sync3", b"done", expected) + frames1 = self._sample_frames( + client_socket, unwinder, b"sync1", b"ack", expected + ) + frames2 = self._sample_frames( + client_socket, unwinder, b"sync2", b"ack", expected + ) + frames3 = self._sample_frames( + client_socket, unwinder, b"sync3", b"done", expected + ) self.assertIsNotNone(frames1) self.assertIsNotNone(frames2) @@ -2176,8 +2364,12 @@ def level1(): # Parent frames (index 1+) must be identical objects (cache reuse) for i in range(1, len(frames1)): f1, f2, f3 = frames1[i], frames2[i], frames3[i] - self.assertIs(f1, f2, f"Frame {i}: samples 1-2 must be same object") - self.assertIs(f2, f3, f"Frame {i}: samples 2-3 must be same object") + self.assertIs( + f1, f2, f"Frame {i}: samples 1-2 must be same object" + ) + self.assertIs( + f2, f3, f"Frame {i}: samples 2-3 must be same object" + ) @skip_if_not_supported @unittest.skipIf( @@ -2203,13 +2395,25 @@ def inner(): outer() """ - with self._target_process(script_body) as (p, client_socket, make_unwinder): + with self._target_process(script_body) as ( + p, + client_socket, + make_unwinder, + ): unwinder = make_unwinder(cache_frames=True) - frames_a = self._sample_frames(client_socket, unwinder, b"line_a", b"ack", {"inner"}) - frames_b = self._sample_frames(client_socket, unwinder, b"line_b", b"ack", {"inner"}) - frames_c = self._sample_frames(client_socket, unwinder, b"line_c", b"ack", {"inner"}) - frames_d = self._sample_frames(client_socket, unwinder, b"line_d", b"done", {"inner"}) + frames_a = self._sample_frames( + client_socket, unwinder, b"line_a", b"ack", {"inner"} + ) + frames_b = self._sample_frames( + client_socket, unwinder, b"line_b", b"ack", {"inner"} + ) + frames_c = self._sample_frames( + client_socket, unwinder, b"line_c", b"ack", {"inner"} + ) + frames_d = self._sample_frames( + client_socket, unwinder, b"line_d", b"done", {"inner"} + ) self.assertIsNotNone(frames_a) self.assertIsNotNone(frames_b) @@ -2228,12 +2432,15 @@ def inner(): self.assertEqual(inner_d.funcname, "inner") # Line numbers must be different and increasing (execution moves forward) - self.assertLess(inner_a.lineno, inner_b.lineno, - "Line B should be after line A") - self.assertLess(inner_b.lineno, inner_c.lineno, - "Line C should be after line B") - self.assertLess(inner_c.lineno, inner_d.lineno, - "Line D should be after line C") + self.assertLess( + inner_a.lineno, inner_b.lineno, "Line B should be after line A" + ) + self.assertLess( + inner_b.lineno, inner_c.lineno, "Line C should be after line B" + ) + self.assertLess( + inner_c.lineno, inner_d.lineno, "Line D should be after line C" + ) @skip_if_not_supported @unittest.skipIf( @@ -2255,13 +2462,23 @@ def outer(): outer() """ - with self._target_process(script_body) as (p, client_socket, make_unwinder): + with self._target_process(script_body) as ( + p, + client_socket, + make_unwinder, + ): unwinder = make_unwinder(cache_frames=True) frames_deep = self._sample_frames( - client_socket, unwinder, b"at_inner", b"ack", {"inner", "outer"}) + client_socket, + unwinder, + b"at_inner", + b"ack", + {"inner", "outer"}, + ) frames_shallow = self._sample_frames( - client_socket, unwinder, b"at_outer", b"done", {"outer"}) + client_socket, unwinder, b"at_outer", b"done", {"outer"} + ) self.assertIsNotNone(frames_deep) self.assertIsNotNone(frames_shallow) @@ -2297,13 +2514,27 @@ def top(): top() """ - with self._target_process(script_body) as (p, client_socket, make_unwinder): + with self._target_process(script_body) as ( + p, + client_socket, + make_unwinder, + ): unwinder = make_unwinder(cache_frames=True) frames_before = self._sample_frames( - client_socket, unwinder, b"at_middle", b"ack", {"middle", "top"}) + client_socket, + unwinder, + b"at_middle", + b"ack", + {"middle", "top"}, + ) frames_after = self._sample_frames( - client_socket, unwinder, b"at_deeper", b"done", {"deeper", "middle", "top"}) + client_socket, + unwinder, + b"at_deeper", + b"done", + {"deeper", "middle", "top"}, + ) self.assertIsNotNone(frames_before) self.assertIsNotNone(frames_after) @@ -2345,15 +2576,29 @@ def func_a(): func_a() """ - with self._target_process(script_body) as (p, client_socket, make_unwinder): + with self._target_process(script_body) as ( + p, + client_socket, + make_unwinder, + ): unwinder = make_unwinder(cache_frames=True) # Sample at C: stack is A→B→C frames_c = self._sample_frames( - client_socket, unwinder, b"at_c", b"ack", {"func_a", "func_b", "func_c"}) + client_socket, + unwinder, + b"at_c", + b"ack", + {"func_a", "func_b", "func_c"}, + ) # Sample at D: stack is A→B→D (C returned, D called) frames_d = self._sample_frames( - client_socket, unwinder, b"at_d", b"done", {"func_a", "func_b", "func_d"}) + client_socket, + unwinder, + b"at_d", + b"done", + {"func_a", "func_b", "func_d"}, + ) self.assertIsNotNone(frames_c) self.assertIsNotNone(frames_d) @@ -2376,8 +2621,16 @@ def find_frame(frames, funcname): self.assertIsNotNone(frame_b_in_d) # The bottom frames (A, B) should be the SAME objects (cache reuse) - self.assertIs(frame_a_in_c, frame_a_in_d, "func_a frame should be reused from cache") - self.assertIs(frame_b_in_c, frame_b_in_d, "func_b frame should be reused from cache") + self.assertIs( + frame_a_in_c, + frame_a_in_d, + "func_a frame should be reused from cache", + ) + self.assertIs( + frame_b_in_c, + frame_b_in_d, + "func_b frame should be reused from cache", + ) @skip_if_not_supported @unittest.skipIf( @@ -2399,13 +2652,19 @@ def recurse(n): recurse(5) """ - with self._target_process(script_body) as (p, client_socket, make_unwinder): + with self._target_process(script_body) as ( + p, + client_socket, + make_unwinder, + ): unwinder = make_unwinder(cache_frames=True) frames1 = self._sample_frames( - client_socket, unwinder, b"sync1", b"ack", {"recurse"}) + client_socket, unwinder, b"sync1", b"ack", {"recurse"} + ) frames2 = self._sample_frames( - client_socket, unwinder, b"sync2", b"done", {"recurse"}) + client_socket, unwinder, b"sync2", b"done", {"recurse"} + ) self.assertIsNotNone(frames1) self.assertIsNotNone(frames2) @@ -2421,8 +2680,11 @@ def recurse(n): # Parent frames (index 1+) should be identical objects (cache reuse) for i in range(1, len(frames1)): - self.assertIs(frames1[i], frames2[i], - f"Frame {i}: recursive frames must be same object") + self.assertIs( + frames1[i], + frames2[i], + f"Frame {i}: recursive frames must be same object", + ) @skip_if_not_supported @unittest.skipIf( @@ -2444,16 +2706,24 @@ def level1(): level1() """ - with self._target_process(script_body) as (p, client_socket, make_unwinder): - self._wait_for_signal(client_socket, b"ready") + with self._target_process(script_body) as ( + p, + client_socket, + make_unwinder, + ): + _wait_for_signal(client_socket, b"ready") # Sample with cache unwinder_cache = make_unwinder(cache_frames=True) - frames_cached = self._get_frames(unwinder_cache, {"level1", "level2", "level3"}) + frames_cached = self._get_frames_with_retry( + unwinder_cache, {"level1", "level2", "level3"} + ) # Sample without cache unwinder_no_cache = make_unwinder(cache_frames=False) - frames_no_cache = self._get_frames(unwinder_no_cache, {"level1", "level2", "level3"}) + frames_no_cache = self._get_frames_with_retry( + unwinder_no_cache, {"level1", "level2", "level3"} + ) client_socket.sendall(b"done") @@ -2526,7 +2796,11 @@ def foo2(): t2.join() """ - with self._target_process(script_body) as (p, client_socket, make_unwinder): + with self._target_process(script_body) as ( + p, + client_socket, + make_unwinder, + ): unwinder = make_unwinder(cache_frames=True) buffer = b"" @@ -2624,16 +2898,24 @@ def get_thread_frames(target_funcs): # Thread 1 at blech1: bar1/baz1 should be GONE (cache invalidated) self.assertIn("blech1", t1_blech) self.assertIn("foo1", t1_blech) - self.assertNotIn("bar1", t1_blech, "Cache not invalidated: bar1 still present") - self.assertNotIn("baz1", t1_blech, "Cache not invalidated: baz1 still present") + self.assertNotIn( + "bar1", t1_blech, "Cache not invalidated: bar1 still present" + ) + self.assertNotIn( + "baz1", t1_blech, "Cache not invalidated: baz1 still present" + ) # No cross-contamination self.assertNotIn("blech2", t1_blech) # Thread 2 at blech2: bar2/baz2 should be GONE (cache invalidated) self.assertIn("blech2", t2_blech) self.assertIn("foo2", t2_blech) - self.assertNotIn("bar2", t2_blech, "Cache not invalidated: bar2 still present") - self.assertNotIn("baz2", t2_blech, "Cache not invalidated: baz2 still present") + self.assertNotIn( + "bar2", t2_blech, "Cache not invalidated: bar2 still present" + ) + self.assertNotIn( + "baz2", t2_blech, "Cache not invalidated: baz2 still present" + ) # No cross-contamination self.assertNotIn("blech1", t2_blech) @@ -2663,17 +2945,25 @@ def level1(): level1() """ - with self._target_process(script_body) as (p, client_socket, make_unwinder): + with self._target_process(script_body) as ( + p, + client_socket, + make_unwinder, + ): expected = {"level1", "level2", "level3", "level4"} # First unwinder samples - this sets last_profiled_frame in target unwinder1 = make_unwinder(cache_frames=True) - frames1 = self._sample_frames(client_socket, unwinder1, b"sync1", b"ack", expected) + frames1 = self._sample_frames( + client_socket, unwinder1, b"sync1", b"ack", expected + ) # Create NEW unwinder (empty cache) and sample # The target still has last_profiled_frame set from unwinder1 unwinder2 = make_unwinder(cache_frames=True) - frames2 = self._sample_frames(client_socket, unwinder2, b"sync2", b"done", expected) + frames2 = self._sample_frames( + client_socket, unwinder2, b"sync2", b"done", expected + ) self.assertIsNotNone(frames1) self.assertIsNotNone(frames2) @@ -2687,8 +2977,11 @@ def level1(): self.assertIn(level, funcs2, f"{level} missing from second sample") # Should have same stack depth - self.assertEqual(len(frames1), len(frames2), - "New unwinder should return complete stack despite stale last_profiled_frame") + self.assertEqual( + len(frames1), + len(frames2), + "New unwinder should return complete stack despite stale last_profiled_frame", + ) @skip_if_not_supported @unittest.skipIf( @@ -2719,16 +3012,30 @@ def recurse(n): recurse({depth}) """ - with self._target_process(script_body) as (p, client_socket, make_unwinder): + with self._target_process(script_body) as ( + p, + client_socket, + make_unwinder, + ): unwinder_cache = make_unwinder(cache_frames=True) unwinder_no_cache = make_unwinder(cache_frames=False) frames_cached = self._sample_frames( - client_socket, unwinder_cache, b"ready", b"ack", {"recurse"}, expected_frames=1102 + client_socket, + unwinder_cache, + b"ready", + b"ack", + {"recurse"}, + expected_frames=1102, ) # Sample again with no cache for comparison frames_no_cache = self._sample_frames( - client_socket, unwinder_no_cache, b"ready2", b"done", {"recurse"}, expected_frames=1102 + client_socket, + unwinder_no_cache, + b"ready2", + b"done", + {"recurse"}, + expected_frames=1102, ) self.assertIsNotNone(frames_cached) @@ -2738,12 +3045,19 @@ def recurse(n): cached_count = [f.funcname for f in frames_cached].count("recurse") no_cache_count = [f.funcname for f in frames_no_cache].count("recurse") - self.assertGreater(cached_count, 1000, "Should have >1000 recurse frames") - self.assertGreater(no_cache_count, 1000, "Should have >1000 recurse frames") + self.assertGreater( + cached_count, 1000, "Should have >1000 recurse frames" + ) + self.assertGreater( + no_cache_count, 1000, "Should have >1000 recurse frames" + ) # Both modes should produce same frame count - self.assertEqual(len(frames_cached), len(frames_no_cache), - "Cache exhaustion should not affect stack completeness") + self.assertEqual( + len(frames_cached), + len(frames_no_cache), + "Cache exhaustion should not affect stack completeness", + ) @skip_if_not_supported @unittest.skipIf( @@ -2759,7 +3073,7 @@ def test_get_stats(self): with self._target_process(script_body) as (p, client_socket, _): unwinder = RemoteUnwinder(p.pid, all_threads=True, stats=True) - self._wait_for_signal(client_socket, b"ready") + _wait_for_signal(client_socket, b"ready") # Take a sample unwinder.get_stack_trace() @@ -2769,14 +3083,18 @@ def test_get_stats(self): # Verify expected keys exist expected_keys = [ - 'total_samples', 'frame_cache_hits', 'frame_cache_misses', - 'frame_cache_partial_hits', 'frames_read_from_cache', - 'frames_read_from_memory', 'frame_cache_hit_rate' + "total_samples", + "frame_cache_hits", + "frame_cache_misses", + "frame_cache_partial_hits", + "frames_read_from_cache", + "frames_read_from_memory", + "frame_cache_hit_rate", ] for key in expected_keys: self.assertIn(key, stats) - self.assertEqual(stats['total_samples'], 1) + self.assertEqual(stats["total_samples"], 1) @skip_if_not_supported @unittest.skipIf( @@ -2791,8 +3109,10 @@ def test_get_stats_disabled_raises(self): """ with self._target_process(script_body) as (p, client_socket, _): - unwinder = RemoteUnwinder(p.pid, all_threads=True) # stats=False by default - self._wait_for_signal(client_socket, b"ready") + unwinder = RemoteUnwinder( + p.pid, all_threads=True + ) # stats=False by default + _wait_for_signal(client_socket, b"ready") with self.assertRaises(RuntimeError): unwinder.get_stats() From c5b37228af6738d3688798d7b48a9f94b21ab028 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 7 Dec 2025 02:46:33 +0000 Subject: [PATCH 289/638] gh-138122: Improve the profiling section in the 3.15 what's new document (#140156) --- Doc/whatsnew/3.15.rst | 173 +++++++++++++++++++----------------------- 1 file changed, 78 insertions(+), 95 deletions(-) diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 27e3f23e47c875..1bd82545e588fa 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -66,7 +66,7 @@ Summary -- Release highlights .. PEP-sized items next. * :pep:`799`: :ref:`A dedicated profiling package for organizing Python - profiling tools ` + profiling tools ` * :pep:`686`: :ref:`Python now uses UTF-8 as the default encoding ` * :pep:`782`: :ref:`A new PyBytesWriter C API to create a Python bytes object @@ -77,12 +77,32 @@ Summary -- Release highlights New features ============ +.. _whatsnew315-profiling-package: + +:pep:`799`: A dedicated profiling package +----------------------------------------- + +A new :mod:`!profiling` module has been added to organize Python's built-in +profiling tools under a single, coherent namespace. This module contains: + +* :mod:`!profiling.tracing`: deterministic function-call tracing (relocated from + :mod:`cProfile`). +* :mod:`!profiling.sampling`: a new statistical sampling profiler (named Tachyon). + +The :mod:`cProfile` module remains as an alias for backwards compatibility. +The :mod:`profile` module is deprecated and will be removed in Python 3.17. + +.. seealso:: :pep:`799` for further details. + +(Contributed by Pablo Galindo and László Kiss Kollár in :gh:`138122`.) + + .. _whatsnew315-sampling-profiler: -:pep:`799`: High frequency statistical sampling profiler --------------------------------------------------------- +Tachyon: High frequency statistical sampling profiler +----------------------------------------------------- -A new statistical sampling profiler has been added to the new :mod:`!profiling` module as +A new statistical sampling profiler (Tachyon) has been added as :mod:`!profiling.sampling`. This profiler enables low-overhead performance analysis of running Python processes without requiring code modification or process restart. @@ -91,101 +111,64 @@ every function call, the sampling profiler periodically captures stack traces fr running processes. This approach provides virtually zero overhead while achieving sampling rates of **up to 1,000,000 Hz**, making it the fastest sampling profiler available for Python (at the time of its contribution) and ideal for debugging -performance issues in production environments. +performance issues in production environments. This capability is particularly +valuable for debugging performance issues in production systems where traditional +profiling approaches would be too intrusive. Key features include: * **Zero-overhead profiling**: Attach to any running Python process without - affecting its performance -* **No code modification required**: Profile existing applications without restart -* **Real-time statistics**: Monitor sampling quality during data collection -* **Multiple output formats**: Generate both detailed statistics and flamegraph data -* **Thread-aware profiling**: Option to profile all threads or just the main thread - -Profile process 1234 for 10 seconds with default settings: - -.. code-block:: shell - - python -m profiling.sampling 1234 - -Profile with custom interval and duration, save to file: - -.. code-block:: shell - - python -m profiling.sampling -i 50 -d 30 -o profile.stats 1234 - -Generate collapsed stacks for flamegraph: - -.. code-block:: shell - - python -m profiling.sampling --collapsed 1234 - -Profile all threads and sort by total time: - -.. code-block:: shell - - python -m profiling.sampling -a --sort-tottime 1234 - -The profiler generates statistical estimates of where time is spent: - -.. code-block:: text - - Real-time sampling stats: Mean: 100261.5Hz (9.97µs) Min: 86333.4Hz (11.58µs) Max: 118807.2Hz (8.42µs) Samples: 400001 - Captured 498841 samples in 5.00 seconds - Sample rate: 99768.04 samples/sec - Error rate: 0.72% - Profile Stats: - nsamples sample% tottime (s) cumul% cumtime (s) filename:lineno(function) - 43/418858 0.0 0.000 87.9 4.189 case.py:667(TestCase.run) - 3293/418812 0.7 0.033 87.9 4.188 case.py:613(TestCase._callTestMethod) - 158562/158562 33.3 1.586 33.3 1.586 test_compile.py:725(TestSpecifics.test_compiler_recursion_limit..check_limit) - 129553/129553 27.2 1.296 27.2 1.296 ast.py:46(parse) - 0/128129 0.0 0.000 26.9 1.281 test_ast.py:884(AST_Tests.test_ast_recursion_limit..check_limit) - 7/67446 0.0 0.000 14.2 0.674 test_compile.py:729(TestSpecifics.test_compiler_recursion_limit) - 6/60380 0.0 0.000 12.7 0.604 test_ast.py:888(AST_Tests.test_ast_recursion_limit) - 3/50020 0.0 0.000 10.5 0.500 test_compile.py:727(TestSpecifics.test_compiler_recursion_limit) - 1/38011 0.0 0.000 8.0 0.380 test_ast.py:886(AST_Tests.test_ast_recursion_limit) - 1/25076 0.0 0.000 5.3 0.251 test_compile.py:728(TestSpecifics.test_compiler_recursion_limit) - 22361/22362 4.7 0.224 4.7 0.224 test_compile.py:1368(TestSpecifics.test_big_dict_literal) - 4/18008 0.0 0.000 3.8 0.180 test_ast.py:889(AST_Tests.test_ast_recursion_limit) - 11/17696 0.0 0.000 3.7 0.177 subprocess.py:1038(Popen.__init__) - 16968/16968 3.6 0.170 3.6 0.170 subprocess.py:1900(Popen._execute_child) - 2/16941 0.0 0.000 3.6 0.169 test_compile.py:730(TestSpecifics.test_compiler_recursion_limit) - - Legend: - nsamples: Direct/Cumulative samples (direct executing / on call stack) - sample%: Percentage of total samples this function was directly executing - tottime: Estimated total time spent directly in this function - cumul%: Percentage of total samples when this function was on the call stack - cumtime: Estimated cumulative time (including time in called functions) - filename:lineno(function): Function location and name - - Summary of Interesting Functions: - - Functions with Highest Direct/Cumulative Ratio (Hot Spots): - 1.000 direct/cumulative ratio, 33.3% direct samples: test_compile.py:(TestSpecifics.test_compiler_recursion_limit..check_limit) - 1.000 direct/cumulative ratio, 27.2% direct samples: ast.py:(parse) - 1.000 direct/cumulative ratio, 3.6% direct samples: subprocess.py:(Popen._execute_child) - - Functions with Highest Call Frequency (Indirect Calls): - 418815 indirect calls, 87.9% total stack presence: case.py:(TestCase.run) - 415519 indirect calls, 87.9% total stack presence: case.py:(TestCase._callTestMethod) - 159470 indirect calls, 33.5% total stack presence: test_compile.py:(TestSpecifics.test_compiler_recursion_limit) - - Functions with Highest Call Magnification (Cumulative/Direct): - 12267.9x call magnification, 159470 indirect calls from 13 direct: test_compile.py:(TestSpecifics.test_compiler_recursion_limit) - 10581.7x call magnification, 116388 indirect calls from 11 direct: test_ast.py:(AST_Tests.test_ast_recursion_limit) - 9740.9x call magnification, 418815 indirect calls from 43 direct: case.py:(TestCase.run) - -The profiler automatically identifies performance bottlenecks through statistical -analysis, highlighting functions with high CPU usage and call frequency patterns. - -This capability is particularly valuable for debugging performance issues in -production systems where traditional profiling approaches would be too intrusive. - - .. seealso:: :pep:`799` for further details. - -(Contributed by Pablo Galindo and László Kiss Kollár in :gh:`135953`.) + affecting its performance. Ideal for production debugging where you can't afford + to restart or slow down your application. + +* **No code modification required**: Profile existing applications without restart. + Simply point the profiler at a running process by PID and start collecting data. + +* **Flexible target modes**: + + * Profile running processes by PID (``attach``) - attach to already-running applications + * Run and profile scripts directly (``run``) - profile from the very start of execution + * Execute and profile modules (``run -m``) - profile packages run as ``python -m module`` + +* **Multiple profiling modes**: Choose what to measure based on your performance investigation: + + * **Wall-clock time** (``--mode wall``, default): Measures real elapsed time including I/O, + network waits, and blocking operations. Use this to understand where your program spends + calendar time, including when waiting for external resources. + * **CPU time** (``--mode cpu``): Measures only active CPU execution time, excluding I/O waits + and blocking. Use this to identify CPU-bound bottlenecks and optimize computational work. + * **GIL-holding time** (``--mode gil``): Measures time spent holding Python's Global Interpreter + Lock. Use this to identify which threads dominate GIL usage in multi-threaded applications. + +* **Thread-aware profiling**: Option to profile all threads (``-a``) or just the main thread, + essential for understanding multi-threaded application behavior. + +* **Multiple output formats**: Choose the visualization that best fits your workflow: + + * ``--pstats``: Detailed tabular statistics compatible with :mod:`pstats`. Shows function-level + timing with direct and cumulative samples. Best for detailed analysis and integration with + existing Python profiling tools. + * ``--collapsed``: Generates collapsed stack traces (one line per stack). This format is + specifically designed for creating flamegraphs with external tools like Brendan Gregg's + FlameGraph scripts or speedscope. + * ``--flamegraph``: Generates a self-contained interactive HTML flamegraph using D3.js. + Opens directly in your browser for immediate visual analysis. Flamegraphs show the call + hierarchy where width represents time spent, making it easy to spot bottlenecks at a glance. + * ``--gecko``: Generates Gecko Profiler format compatible with Firefox Profiler + (https://profiler.firefox.com). Upload the output to Firefox Profiler for advanced + timeline-based analysis with features like stack charts, markers, and network activity. + * ``--heatmap``: Generates an interactive HTML heatmap visualization with line-level sample + counts. Creates a directory with per-file heatmaps showing exactly where time is spent + at the source code level. + +* **Live interactive mode**: Real-time TUI profiler with a top-like interface (``--live``). + Monitor performance as your application runs with interactive sorting and filtering. + +* **Async-aware profiling**: Profile async/await code with task-based stack reconstruction + (``--async-aware``). See which coroutines are consuming time, with options to show only + running tasks or all tasks including those waiting. + +(Contributed by Pablo Galindo and László Kiss Kollár in :gh:`135953` and :gh:`138122`.) .. _whatsnew315-improved-error-messages: From d6d850df89c97fbced893b7914682efedfa6ebc4 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 7 Dec 2025 15:53:48 +0000 Subject: [PATCH 290/638] gh-138122: Don't sample partial frame chains (#141912) --- Include/cpython/pystate.h | 7 + Include/internal/pycore_debug_offsets.h | 2 + Include/internal/pycore_tstate.h | 5 + InternalDocs/frames.md | 29 ++++ Lib/test/test_external_inspection.py | 126 +++++------------- ...-11-24-16-07-57.gh-issue-138122.m3EF9E.rst | 6 + Modules/_remote_debugging/_remote_debugging.h | 1 + Modules/_remote_debugging/frames.c | 30 +++-- Modules/_remote_debugging/threads.c | 5 +- Python/ceval.c | 3 + Python/pylifecycle.c | 2 +- Python/pystate.c | 28 +++- Python/traceback.c | 2 +- 13 files changed, 138 insertions(+), 108 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-24-16-07-57.gh-issue-138122.m3EF9E.rst diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 08d71070ddccd3..22df26bd37a5c5 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -135,6 +135,13 @@ struct _ts { /* Pointer to currently executing frame. */ struct _PyInterpreterFrame *current_frame; + /* Pointer to the base frame (bottommost sentinel frame). + Used by profilers to validate complete stack unwinding. + Points to the embedded base_frame in _PyThreadStateImpl. + The frame is embedded there rather than here because _PyInterpreterFrame + is defined in internal headers that cannot be exposed in the public API. */ + struct _PyInterpreterFrame *base_frame; + struct _PyInterpreterFrame *last_profiled_frame; Py_tracefunc c_profilefunc; diff --git a/Include/internal/pycore_debug_offsets.h b/Include/internal/pycore_debug_offsets.h index bfd86c08887b08..1cdc4449b173e8 100644 --- a/Include/internal/pycore_debug_offsets.h +++ b/Include/internal/pycore_debug_offsets.h @@ -102,6 +102,7 @@ typedef struct _Py_DebugOffsets { uint64_t next; uint64_t interp; uint64_t current_frame; + uint64_t base_frame; uint64_t last_profiled_frame; uint64_t thread_id; uint64_t native_thread_id; @@ -273,6 +274,7 @@ typedef struct _Py_DebugOffsets { .next = offsetof(PyThreadState, next), \ .interp = offsetof(PyThreadState, interp), \ .current_frame = offsetof(PyThreadState, current_frame), \ + .base_frame = offsetof(PyThreadState, base_frame), \ .last_profiled_frame = offsetof(PyThreadState, last_profiled_frame), \ .thread_id = offsetof(PyThreadState, thread_id), \ .native_thread_id = offsetof(PyThreadState, native_thread_id), \ diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index 50048801b2e4ee..c4f723ac8abbbe 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -10,6 +10,7 @@ extern "C" { #include "pycore_brc.h" // struct _brc_thread_state #include "pycore_freelist_state.h" // struct _Py_freelists +#include "pycore_interpframe_structs.h" // _PyInterpreterFrame #include "pycore_mimalloc.h" // struct _mimalloc_thread_state #include "pycore_qsbr.h" // struct qsbr #include "pycore_uop.h" // struct _PyUOpInstruction @@ -61,6 +62,10 @@ typedef struct _PyThreadStateImpl { // semi-public fields are in PyThreadState. PyThreadState base; + // Embedded base frame - sentinel at the bottom of the frame stack. + // Used by profiling/sampling to detect incomplete stack traces. + _PyInterpreterFrame base_frame; + // The reference count field is used to synchronize deallocation of the // thread state during runtime finalization. Py_ssize_t refcount; diff --git a/InternalDocs/frames.md b/InternalDocs/frames.md index d56e5481d3cbfe..60ab2055afa7b1 100644 --- a/InternalDocs/frames.md +++ b/InternalDocs/frames.md @@ -111,6 +111,35 @@ The shim frame points to a special code object containing the `INTERPRETER_EXIT` instruction which cleans up the shim frame and returns. +### Base frame + +Each thread state contains an embedded `_PyInterpreterFrame` called the "base frame" +that serves as a sentinel at the bottom of the frame stack. This frame is allocated +in `_PyThreadStateImpl` (the internal extension of `PyThreadState`) and initialized +when the thread state is created. The `owner` field is set to `FRAME_OWNED_BY_INTERPRETER`. + +External profilers and sampling tools can validate that they have successfully unwound +the complete call stack by checking that the frame chain terminates at the base frame. +The `PyThreadState.base_frame` pointer provides the expected address to compare against. +If a stack walk doesn't reach this frame, the sample is incomplete (possibly due to a +race condition) and should be discarded. + +The base frame is embedded in `_PyThreadStateImpl` rather than `PyThreadState` because +`_PyInterpreterFrame` is defined in internal headers that cannot be exposed in the +public API. A pointer (`PyThreadState.base_frame`) is provided for profilers to access +the address without needing internal headers. + +See the initialization in `new_threadstate()` in [Python/pystate.c](../Python/pystate.c). + +#### How profilers should use the base frame + +External profilers should read `tstate->base_frame` before walking the stack, then +walk from `tstate->current_frame` following `frame->previous` pointers until reaching +a frame with `owner == FRAME_OWNED_BY_INTERPRETER`. After the walk, verify that the +last frame address matches `base_frame`. If not, discard the sample as incomplete +since the frame chain may have been in an inconsistent state due to concurrent updates. + + ### Remote Profiling Frame Cache The `last_profiled_frame` field in `PyThreadState` supports an optimization for diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index f664e8ac53ff95..a97242483a8942 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -1,7 +1,7 @@ -import contextlib import unittest import os import textwrap +import contextlib import importlib import sys import socket @@ -216,33 +216,13 @@ def requires_subinterpreters(meth): # Simple wrapper functions for RemoteUnwinder # ============================================================================ -# Errors that can occur transiently when reading process memory without synchronization -RETRIABLE_ERRORS = ( - "Task list appears corrupted", - "Invalid linked list structure reading remote memory", - "Unknown error reading memory", - "Unhandled frame owner", - "Failed to parse initial frame", - "Failed to process frame chain", - "Failed to unwind stack", -) - - -def _is_retriable_error(exc): - """Check if an exception is a transient error that should be retried.""" - msg = str(exc) - return any(msg.startswith(err) or err in msg for err in RETRIABLE_ERRORS) - - def get_stack_trace(pid): for _ in busy_retry(SHORT_TIMEOUT): try: unwinder = RemoteUnwinder(pid, all_threads=True, debug=True) return unwinder.get_stack_trace() except RuntimeError as e: - if _is_retriable_error(e): - continue - raise + continue raise RuntimeError("Failed to get stack trace after retries") @@ -252,9 +232,7 @@ def get_async_stack_trace(pid): unwinder = RemoteUnwinder(pid, debug=True) return unwinder.get_async_stack_trace() except RuntimeError as e: - if _is_retriable_error(e): - continue - raise + continue raise RuntimeError("Failed to get async stack trace after retries") @@ -264,9 +242,7 @@ def get_all_awaited_by(pid): unwinder = RemoteUnwinder(pid, debug=True) return unwinder.get_all_awaited_by() except RuntimeError as e: - if _is_retriable_error(e): - continue - raise + continue raise RuntimeError("Failed to get all awaited_by after retries") @@ -2268,18 +2244,13 @@ def make_unwinder(cache_frames=True): def _get_frames_with_retry(self, unwinder, required_funcs): """Get frames containing required_funcs, with retry for transient errors.""" for _ in range(MAX_TRIES): - try: + with contextlib.suppress(OSError, RuntimeError): traces = unwinder.get_stack_trace() for interp in traces: for thread in interp.threads: funcs = {f.funcname for f in thread.frame_info} if required_funcs.issubset(funcs): return thread.frame_info - except RuntimeError as e: - if _is_retriable_error(e): - pass - else: - raise time.sleep(0.1) return None @@ -2802,70 +2773,39 @@ def foo2(): make_unwinder, ): unwinder = make_unwinder(cache_frames=True) - buffer = b"" - - def recv_msg(): - """Receive a single message from socket.""" - nonlocal buffer - while b"\n" not in buffer: - chunk = client_socket.recv(256) - if not chunk: - return None - buffer += chunk - msg, buffer = buffer.split(b"\n", 1) - return msg - - def get_thread_frames(target_funcs): - """Get frames for thread matching target functions.""" - retries = 0 - for _ in busy_retry(SHORT_TIMEOUT): - if retries >= 5: - break - retries += 1 - # On Windows, ReadProcessMemory can fail with OSError - # (WinError 299) when frame pointers are in flux - with contextlib.suppress(RuntimeError, OSError): - traces = unwinder.get_stack_trace() - for interp in traces: - for thread in interp.threads: - funcs = [f.funcname for f in thread.frame_info] - if any(f in funcs for f in target_funcs): - return funcs - return None + + # Message dispatch table: signal -> required functions for that thread + dispatch = { + b"t1:baz1": {"baz1", "bar1", "foo1"}, + b"t2:baz2": {"baz2", "bar2", "foo2"}, + b"t1:blech1": {"blech1", "foo1"}, + b"t2:blech2": {"blech2", "foo2"}, + } # Track results for each sync point results = {} - # Process 4 sync points: baz1, baz2, blech1, blech2 - # With the lock, threads are serialized - handle one at a time - for _ in range(4): - msg = recv_msg() - self.assertIsNotNone(msg, "Expected message from subprocess") - - # Determine which thread/function and take snapshot - if msg == b"t1:baz1": - funcs = get_thread_frames(["baz1", "bar1", "foo1"]) - self.assertIsNotNone(funcs, "Thread 1 not found at baz1") - results["t1:baz1"] = funcs - elif msg == b"t2:baz2": - funcs = get_thread_frames(["baz2", "bar2", "foo2"]) - self.assertIsNotNone(funcs, "Thread 2 not found at baz2") - results["t2:baz2"] = funcs - elif msg == b"t1:blech1": - funcs = get_thread_frames(["blech1", "foo1"]) - self.assertIsNotNone(funcs, "Thread 1 not found at blech1") - results["t1:blech1"] = funcs - elif msg == b"t2:blech2": - funcs = get_thread_frames(["blech2", "foo2"]) - self.assertIsNotNone(funcs, "Thread 2 not found at blech2") - results["t2:blech2"] = funcs - - # Release thread to continue + # Process 4 sync points (order depends on thread scheduling) + buffer = _wait_for_signal(client_socket, b"\n") + for i in range(4): + # Extract first message from buffer + msg, sep, buffer = buffer.partition(b"\n") + self.assertIn(msg, dispatch, f"Unexpected message: {msg!r}") + + # Sample frames for the thread at this sync point + required_funcs = dispatch[msg] + frames = self._get_frames_with_retry(unwinder, required_funcs) + self.assertIsNotNone(frames, f"Thread not found for {msg!r}") + results[msg] = [f.funcname for f in frames] + + # Release thread and wait for next message (if not last) client_socket.sendall(b"k") + if i < 3: + buffer += _wait_for_signal(client_socket, b"\n") # Validate Phase 1: baz snapshots - t1_baz = results.get("t1:baz1") - t2_baz = results.get("t2:baz2") + t1_baz = results.get(b"t1:baz1") + t2_baz = results.get(b"t2:baz2") self.assertIsNotNone(t1_baz, "Missing t1:baz1 snapshot") self.assertIsNotNone(t2_baz, "Missing t2:baz2 snapshot") @@ -2890,8 +2830,8 @@ def get_thread_frames(target_funcs): self.assertNotIn("foo1", t2_baz) # Validate Phase 2: blech snapshots (cache invalidation test) - t1_blech = results.get("t1:blech1") - t2_blech = results.get("t2:blech2") + t1_blech = results.get(b"t1:blech1") + t2_blech = results.get(b"t2:blech2") self.assertIsNotNone(t1_blech, "Missing t1:blech1 snapshot") self.assertIsNotNone(t2_blech, "Missing t2:blech2 snapshot") diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-24-16-07-57.gh-issue-138122.m3EF9E.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-24-16-07-57.gh-issue-138122.m3EF9E.rst new file mode 100644 index 00000000000000..a4a29e400274cb --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-24-16-07-57.gh-issue-138122.m3EF9E.rst @@ -0,0 +1,6 @@ +Add incomplete sample detection to prevent corrupted profiling data. Each +thread state now contains an embedded base frame (sentinel at the bottom of +the frame stack) with owner type ``FRAME_OWNED_BY_INTERPRETER``. The profiler +validates that stack unwinding terminates at this sentinel frame. Samples that +fail to reach the base frame (due to race conditions, memory corruption, or +other errors) are now rejected rather than being included as spurious data. diff --git a/Modules/_remote_debugging/_remote_debugging.h b/Modules/_remote_debugging/_remote_debugging.h index 804e2c904e147a..7f3c0d363f56c6 100644 --- a/Modules/_remote_debugging/_remote_debugging.h +++ b/Modules/_remote_debugging/_remote_debugging.h @@ -401,6 +401,7 @@ extern int process_frame_chain( uintptr_t initial_frame_addr, StackChunkList *chunks, PyObject *frame_info, + uintptr_t base_frame_addr, uintptr_t gc_frame, uintptr_t last_profiled_frame, int *stopped_at_cached_frame, diff --git a/Modules/_remote_debugging/frames.c b/Modules/_remote_debugging/frames.c index b77c0ca556d5b3..eaf3287c6fec12 100644 --- a/Modules/_remote_debugging/frames.c +++ b/Modules/_remote_debugging/frames.c @@ -154,14 +154,13 @@ is_frame_valid( void* frame = (void*)frame_addr; - if (GET_MEMBER(char, frame, unwinder->debug_offsets.interpreter_frame.owner) == FRAME_OWNED_BY_INTERPRETER) { - return 0; // C frame + char owner = GET_MEMBER(char, frame, unwinder->debug_offsets.interpreter_frame.owner); + if (owner == FRAME_OWNED_BY_INTERPRETER) { + return 0; // C frame or sentinel base frame } - if (GET_MEMBER(char, frame, unwinder->debug_offsets.interpreter_frame.owner) != FRAME_OWNED_BY_GENERATOR - && GET_MEMBER(char, frame, unwinder->debug_offsets.interpreter_frame.owner) != FRAME_OWNED_BY_THREAD) { - PyErr_Format(PyExc_RuntimeError, "Unhandled frame owner %d.\n", - GET_MEMBER(char, frame, unwinder->debug_offsets.interpreter_frame.owner)); + if (owner != FRAME_OWNED_BY_GENERATOR && owner != FRAME_OWNED_BY_THREAD) { + PyErr_Format(PyExc_RuntimeError, "Unhandled frame owner %d.\n", owner); set_exception_cause(unwinder, PyExc_RuntimeError, "Unhandled frame owner type in async frame"); return -1; } @@ -260,6 +259,7 @@ process_frame_chain( uintptr_t initial_frame_addr, StackChunkList *chunks, PyObject *frame_info, + uintptr_t base_frame_addr, uintptr_t gc_frame, uintptr_t last_profiled_frame, int *stopped_at_cached_frame, @@ -269,6 +269,7 @@ process_frame_chain( { uintptr_t frame_addr = initial_frame_addr; uintptr_t prev_frame_addr = 0; + uintptr_t last_frame_addr = 0; // Track last frame visited for validation const size_t MAX_FRAMES = 1024 + 512; size_t frame_count = 0; @@ -296,6 +297,7 @@ process_frame_chain( PyObject *frame = NULL; uintptr_t next_frame_addr = 0; uintptr_t stackpointer = 0; + last_frame_addr = frame_addr; // Remember this frame address if (++frame_count > MAX_FRAMES) { PyErr_SetString(PyExc_RuntimeError, "Too many stack frames (possible infinite loop)"); @@ -303,7 +305,6 @@ process_frame_chain( return -1; } - // Try chunks first, fallback to direct memory read if (parse_frame_from_chunks(unwinder, &frame, frame_addr, &next_frame_addr, &stackpointer, chunks) < 0) { PyErr_Clear(); uintptr_t address_of_code_object = 0; @@ -377,6 +378,17 @@ process_frame_chain( frame_addr = next_frame_addr; } + // Validate we reached the base frame (sentinel at bottom of stack) + // Only validate if we walked the full chain (didn't stop at cached frame) + // and base_frame_addr is provided (non-zero) + int stopped_early = stopped_at_cached_frame && *stopped_at_cached_frame; + if (!stopped_early && base_frame_addr != 0 && last_frame_addr != base_frame_addr) { + PyErr_Format(PyExc_RuntimeError, + "Incomplete sample: did not reach base frame (expected 0x%lx, got 0x%lx)", + base_frame_addr, last_frame_addr); + return -1; + } + return 0; } @@ -540,7 +552,7 @@ collect_frames_with_cache( Py_ssize_t frames_before = PyList_GET_SIZE(frame_info); int stopped_at_cached = 0; - if (process_frame_chain(unwinder, frame_addr, chunks, frame_info, gc_frame, + if (process_frame_chain(unwinder, frame_addr, chunks, frame_info, 0, gc_frame, last_profiled_frame, &stopped_at_cached, addrs, &num_addrs, FRAME_CACHE_MAX_FRAMES) < 0) { return -1; @@ -562,7 +574,7 @@ collect_frames_with_cache( // Cache miss - continue walking from last_profiled_frame to get the rest STATS_INC(unwinder, frame_cache_misses); Py_ssize_t frames_before_walk = PyList_GET_SIZE(frame_info); - if (process_frame_chain(unwinder, last_profiled_frame, chunks, frame_info, gc_frame, + if (process_frame_chain(unwinder, last_profiled_frame, chunks, frame_info, 0, gc_frame, 0, NULL, addrs, &num_addrs, FRAME_CACHE_MAX_FRAMES) < 0) { return -1; } diff --git a/Modules/_remote_debugging/threads.c b/Modules/_remote_debugging/threads.c index ce013f902d1ed7..69819eb8dcd645 100644 --- a/Modules/_remote_debugging/threads.c +++ b/Modules/_remote_debugging/threads.c @@ -380,6 +380,7 @@ unwind_stack_for_thread( } uintptr_t frame_addr = GET_MEMBER(uintptr_t, ts, unwinder->debug_offsets.thread_state.current_frame); + uintptr_t base_frame_addr = GET_MEMBER(uintptr_t, ts, unwinder->debug_offsets.thread_state.base_frame); frame_info = PyList_New(0); if (!frame_info) { @@ -411,9 +412,9 @@ unwind_stack_for_thread( PyErr_Clear(); // Non-fatal } } else { - // No caching - process entire frame chain + // No caching - process entire frame chain with base_frame validation if (process_frame_chain(unwinder, frame_addr, &chunks, frame_info, - gc_frame, 0, NULL, NULL, NULL, 0) < 0) { + base_frame_addr, gc_frame, 0, NULL, NULL, NULL, 0) < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to process frame chain"); goto error; } diff --git a/Python/ceval.c b/Python/ceval.c index 382ae210ebbf2b..a1d54bd058bc49 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -3352,6 +3352,9 @@ PyEval_MergeCompilerFlags(PyCompilerFlags *cf) { PyThreadState *tstate = _PyThreadState_GET(); _PyInterpreterFrame *current_frame = tstate->current_frame; + if (current_frame == tstate->base_frame) { + current_frame = NULL; + } int result = cf->cf_flags != 0; if (current_frame != NULL) { diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 67368b5ce077aa..2527dca71d774e 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -2571,7 +2571,7 @@ Py_EndInterpreter(PyThreadState *tstate) if (tstate != _PyThreadState_GET()) { Py_FatalError("thread is not current"); } - if (tstate->current_frame != NULL) { + if (tstate->current_frame != tstate->base_frame) { Py_FatalError("thread still has a frame"); } interp->finalizing = 1; diff --git a/Python/pystate.c b/Python/pystate.c index c12a1418e74309..2956e785405a0e 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1482,7 +1482,31 @@ init_threadstate(_PyThreadStateImpl *_tstate, // This is cleared when PyGILState_Ensure() creates the thread state. tstate->gilstate_counter = 1; - tstate->current_frame = NULL; + // Initialize the embedded base frame - sentinel at the bottom of the frame stack + _tstate->base_frame.previous = NULL; + _tstate->base_frame.f_executable = PyStackRef_None; + _tstate->base_frame.f_funcobj = PyStackRef_NULL; + _tstate->base_frame.f_globals = NULL; + _tstate->base_frame.f_builtins = NULL; + _tstate->base_frame.f_locals = NULL; + _tstate->base_frame.frame_obj = NULL; + _tstate->base_frame.instr_ptr = NULL; + _tstate->base_frame.stackpointer = _tstate->base_frame.localsplus; + _tstate->base_frame.return_offset = 0; + _tstate->base_frame.owner = FRAME_OWNED_BY_INTERPRETER; + _tstate->base_frame.visited = 0; +#ifdef Py_DEBUG + _tstate->base_frame.lltrace = 0; +#endif +#ifdef Py_GIL_DISABLED + _tstate->base_frame.tlbc_index = 0; +#endif + _tstate->base_frame.localsplus[0] = PyStackRef_NULL; + + // current_frame starts pointing to the base frame + tstate->current_frame = &_tstate->base_frame; + // base_frame pointer for profilers to validate stack unwinding + tstate->base_frame = &_tstate->base_frame; tstate->datastack_chunk = NULL; tstate->datastack_top = NULL; tstate->datastack_limit = NULL; @@ -1660,7 +1684,7 @@ PyThreadState_Clear(PyThreadState *tstate) int verbose = _PyInterpreterState_GetConfig(tstate->interp)->verbose; - if (verbose && tstate->current_frame != NULL) { + if (verbose && tstate->current_frame != tstate->base_frame) { /* bpo-20526: After the main thread calls _PyInterpreterState_SetFinalizing() in Py_FinalizeEx() (or in Py_EndInterpreter() for subinterpreters), diff --git a/Python/traceback.c b/Python/traceback.c index 48f9b4d04c6b1a..8af63c22a9f84e 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -1036,7 +1036,7 @@ static int dump_frame(int fd, _PyInterpreterFrame *frame) { if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { - /* Ignore trampoline frame */ + /* Ignore trampoline frames and base frame sentinel */ return 0; } From 1db9f56bff5bbb0292b131ea8a928612acb7ec16 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 7 Dec 2025 21:36:01 +0200 Subject: [PATCH 291/638] gh-142346: Fix usage formatting for mutually exclusive groups in argparse (GH-142381) Support groups preceded by positional arguments or followed or intermixed with other optional arguments. Support empty groups. --- Lib/argparse.py | 211 ++++++++---------- Lib/test/test_argparse.py | 63 +++--- ...-12-07-17-30-05.gh-issue-142346.okcAAp.rst | 3 + 3 files changed, 131 insertions(+), 146 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-12-07-17-30-05.gh-issue-142346.okcAAp.rst diff --git a/Lib/argparse.py b/Lib/argparse.py index 9e2e076936cb51..6ed7386d0dd407 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -334,31 +334,15 @@ def _format_usage(self, usage, actions, groups, prefix): elif usage is None: prog = '%(prog)s' % dict(prog=self._prog) - # split optionals from positionals - optionals = [] - positionals = [] - for action in actions: - if action.option_strings: - optionals.append(action) - else: - positionals.append(action) - + parts, pos_start = self._get_actions_usage_parts(actions, groups) # build full usage string - format = self._format_actions_usage - action_usage = format(optionals + positionals, groups) - usage = ' '.join([s for s in [prog, action_usage] if s]) + usage = ' '.join(filter(None, [prog, *parts])) # wrap the usage parts if it's too long text_width = self._width - self._current_indent if len(prefix) + len(self._decolor(usage)) > text_width: # break usage into wrappable parts - # keep optionals and positionals together to preserve - # mutually exclusive group formatting (gh-75949) - all_actions = optionals + positionals - parts, pos_start = self._get_actions_usage_parts_with_split( - all_actions, groups, len(optionals) - ) opt_parts = parts[:pos_start] pos_parts = parts[pos_start:] @@ -417,125 +401,114 @@ def get_lines(parts, indent, prefix=None): # prefix with 'usage:' return f'{t.usage}{prefix}{t.reset}{usage}\n\n' - def _format_actions_usage(self, actions, groups): - return ' '.join(self._get_actions_usage_parts(actions, groups)) - def _is_long_option(self, string): return len(string) > 2 def _get_actions_usage_parts(self, actions, groups): - parts, _ = self._get_actions_usage_parts_with_split(actions, groups) - return parts - - def _get_actions_usage_parts_with_split(self, actions, groups, opt_count=None): """Get usage parts with split index for optionals/positionals. Returns (parts, pos_start) where pos_start is the index in parts - where positionals begin. When opt_count is None, pos_start is None. + where positionals begin. This preserves mutually exclusive group formatting across the optionals/positionals boundary (gh-75949). """ - # find group indices and identify actions in groups - group_actions = set() - inserts = {} + actions = [action for action in actions if action.help is not SUPPRESS] + # group actions by mutually exclusive groups + action_groups = dict.fromkeys(actions) for group in groups: - if not group._group_actions: - raise ValueError(f'empty group {group}') - - if all(action.help is SUPPRESS for action in group._group_actions): - continue - - try: - start = min(actions.index(item) for item in group._group_actions) - except ValueError: - continue - else: - end = start + len(group._group_actions) - if set(actions[start:end]) == set(group._group_actions): - group_actions.update(group._group_actions) - inserts[start, end] = group + for action in group._group_actions: + if action in action_groups: + action_groups[action] = group + # positional arguments keep their position + positionals = [] + for action in actions: + if not action.option_strings: + group = action_groups.pop(action) + if group: + group_actions = [ + action2 for action2 in group._group_actions + if action2.option_strings and + action_groups.pop(action2, None) + ] + [action] + positionals.append((group.required, group_actions)) + else: + positionals.append((None, [action])) + # the remaining optional arguments are sorted by the position of + # the first option in the group + optionals = [] + for action in actions: + if action.option_strings and action in action_groups: + group = action_groups.pop(action) + if group: + group_actions = [action] + [ + action2 for action2 in group._group_actions + if action2.option_strings and + action_groups.pop(action2, None) + ] + optionals.append((group.required, group_actions)) + else: + optionals.append((None, [action])) # collect all actions format strings parts = [] t = self._theme - for action in actions: - - # suppressed arguments are marked with None - if action.help is SUPPRESS: - part = None - - # produce all arg strings - elif not action.option_strings: - default = self._get_default_metavar_for_positional(action) - part = self._format_args(action, default) - # if it's in a group, strip the outer [] - if action in group_actions: - if part[0] == '[' and part[-1] == ']': - part = part[1:-1] - part = t.summary_action + part + t.reset - - # produce the first way to invoke the option in brackets - else: - option_string = action.option_strings[0] - if self._is_long_option(option_string): - option_color = t.summary_long_option + pos_start = None + for i, (required, group) in enumerate(optionals + positionals): + start = len(parts) + if i == len(optionals): + pos_start = start + in_group = len(group) > 1 + for action in group: + # produce all arg strings + if not action.option_strings: + default = self._get_default_metavar_for_positional(action) + part = self._format_args(action, default) + # if it's in a group, strip the outer [] + if in_group: + if part[0] == '[' and part[-1] == ']': + part = part[1:-1] + part = t.summary_action + part + t.reset + + # produce the first way to invoke the option in brackets else: - option_color = t.summary_short_option - - # if the Optional doesn't take a value, format is: - # -s or --long - if action.nargs == 0: - part = action.format_usage() - part = f"{option_color}{part}{t.reset}" - - # if the Optional takes a value, format is: - # -s ARGS or --long ARGS - else: - default = self._get_default_metavar_for_optional(action) - args_string = self._format_args(action, default) - part = ( - f"{option_color}{option_string} " - f"{t.summary_label}{args_string}{t.reset}" - ) - - # make it look optional if it's not required or in a group - if not action.required and action not in group_actions: - part = '[%s]' % part + option_string = action.option_strings[0] + if self._is_long_option(option_string): + option_color = t.summary_long_option + else: + option_color = t.summary_short_option - # add the action string to the list - parts.append(part) + # if the Optional doesn't take a value, format is: + # -s or --long + if action.nargs == 0: + part = action.format_usage() + part = f"{option_color}{part}{t.reset}" - # group mutually exclusive actions - inserted_separators_indices = set() - for start, end in sorted(inserts, reverse=True): - group = inserts[start, end] - group_parts = [item for item in parts[start:end] if item is not None] - group_size = len(group_parts) - if group.required: - open, close = "()" if group_size > 1 else ("", "") - else: - open, close = "[]" - group_parts[0] = open + group_parts[0] - group_parts[-1] = group_parts[-1] + close - for i, part in enumerate(group_parts[:-1], start=start): - # insert a separator if not already done in a nested group - if i not in inserted_separators_indices: - parts[i] = part + ' |' - inserted_separators_indices.add(i) - parts[start + group_size - 1] = group_parts[-1] - for i in range(start + group_size, end): - parts[i] = None - - # if opt_count is provided, calculate where positionals start in - # the final parts list (for wrapping onto separate lines). - # Count before filtering None entries since indices shift after. - if opt_count is not None: - pos_start = sum(1 for p in parts[:opt_count] if p is not None) - else: - pos_start = None - - # return the usage parts and split point (gh-75949) - return [item for item in parts if item is not None], pos_start + # if the Optional takes a value, format is: + # -s ARGS or --long ARGS + else: + default = self._get_default_metavar_for_optional(action) + args_string = self._format_args(action, default) + part = ( + f"{option_color}{option_string} " + f"{t.summary_label}{args_string}{t.reset}" + ) + + # make it look optional if it's not required or in a group + if not (action.required or required or in_group): + part = '[%s]' % part + + # add the action string to the list + parts.append(part) + + if in_group: + parts[start] = ('(' if required else '[') + parts[start] + for i in range(start, len(parts) - 1): + parts[i] += ' |' + parts[-1] += ')' if required else ']' + + if pos_start is None: + pos_start = len(parts) + return parts, pos_start def _format_text(self, text): if '%(prog)' in text: diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 248a92db74eb69..fe5232e9f3b591 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -3398,12 +3398,11 @@ def test_help_subparser_all_mutually_exclusive_group_members_suppressed(self): ''' self.assertEqual(cmd_foo.format_help(), textwrap.dedent(expected)) - def test_empty_group(self): + def test_usage_empty_group(self): # See issue 26952 - parser = argparse.ArgumentParser() + parser = ErrorRaisingArgumentParser(prog='PROG') group = parser.add_mutually_exclusive_group() - with self.assertRaises(ValueError): - parser.parse_args(['-h']) + self.assertEqual(parser.format_usage(), 'usage: PROG [-h]\n') def test_nested_mutex_groups(self): parser = argparse.ArgumentParser(prog='PROG') @@ -3671,25 +3670,29 @@ def get_parser(self, required): group.add_argument('-b', action='store_true', help='b help') parser.add_argument('-y', action='store_true', help='y help') group.add_argument('-c', action='store_true', help='c help') + parser.add_argument('-z', action='store_true', help='z help') return parser failures = ['-a -b', '-b -c', '-a -c', '-a -b -c'] successes = [ - ('-a', NS(a=True, b=False, c=False, x=False, y=False)), - ('-b', NS(a=False, b=True, c=False, x=False, y=False)), - ('-c', NS(a=False, b=False, c=True, x=False, y=False)), - ('-a -x', NS(a=True, b=False, c=False, x=True, y=False)), - ('-y -b', NS(a=False, b=True, c=False, x=False, y=True)), - ('-x -y -c', NS(a=False, b=False, c=True, x=True, y=True)), + ('-a', NS(a=True, b=False, c=False, x=False, y=False, z=False)), + ('-b', NS(a=False, b=True, c=False, x=False, y=False, z=False)), + ('-c', NS(a=False, b=False, c=True, x=False, y=False, z=False)), + ('-a -x', NS(a=True, b=False, c=False, x=True, y=False, z=False)), + ('-y -b', NS(a=False, b=True, c=False, x=False, y=True, z=False)), + ('-x -y -c', NS(a=False, b=False, c=True, x=True, y=True, z=False)), ] successes_when_not_required = [ - ('', NS(a=False, b=False, c=False, x=False, y=False)), - ('-x', NS(a=False, b=False, c=False, x=True, y=False)), - ('-y', NS(a=False, b=False, c=False, x=False, y=True)), + ('', NS(a=False, b=False, c=False, x=False, y=False, z=False)), + ('-x', NS(a=False, b=False, c=False, x=True, y=False, z=False)), + ('-y', NS(a=False, b=False, c=False, x=False, y=True, z=False)), ] - usage_when_required = usage_when_not_required = '''\ - usage: PROG [-h] [-x] [-a] [-b] [-y] [-c] + usage_when_not_required = '''\ + usage: PROG [-h] [-x] [-a | -b | -c] [-y] [-z] + ''' + usage_when_required = '''\ + usage: PROG [-h] [-x] (-a | -b | -c) [-y] [-z] ''' help = '''\ @@ -3700,6 +3703,7 @@ def get_parser(self, required): -b b help -y y help -c c help + -z z help ''' @@ -3753,23 +3757,27 @@ def get_parser(self, required): group.add_argument('a', nargs='?', help='a help') group.add_argument('-b', action='store_true', help='b help') group.add_argument('-c', action='store_true', help='c help') + parser.add_argument('-z', action='store_true', help='z help') return parser failures = ['X A -b', '-b -c', '-c X A'] successes = [ - ('X A', NS(a='A', b=False, c=False, x='X', y=False)), - ('X -b', NS(a=None, b=True, c=False, x='X', y=False)), - ('X -c', NS(a=None, b=False, c=True, x='X', y=False)), - ('X A -y', NS(a='A', b=False, c=False, x='X', y=True)), - ('X -y -b', NS(a=None, b=True, c=False, x='X', y=True)), + ('X A', NS(a='A', b=False, c=False, x='X', y=False, z=False)), + ('X -b', NS(a=None, b=True, c=False, x='X', y=False, z=False)), + ('X -c', NS(a=None, b=False, c=True, x='X', y=False, z=False)), + ('X A -y', NS(a='A', b=False, c=False, x='X', y=True, z=False)), + ('X -y -b', NS(a=None, b=True, c=False, x='X', y=True, z=False)), ] successes_when_not_required = [ - ('X', NS(a=None, b=False, c=False, x='X', y=False)), - ('X -y', NS(a=None, b=False, c=False, x='X', y=True)), + ('X', NS(a=None, b=False, c=False, x='X', y=False, z=False)), + ('X -y', NS(a=None, b=False, c=False, x='X', y=True, z=False)), ] - usage_when_required = usage_when_not_required = '''\ - usage: PROG [-h] [-y] [-b] [-c] x [a] + usage_when_not_required = '''\ + usage: PROG [-h] [-y] [-z] x [-b | -c | a] + ''' + usage_when_required = '''\ + usage: PROG [-h] [-y] [-z] x (-b | -c | a) ''' help = '''\ @@ -3782,6 +3790,7 @@ def get_parser(self, required): -y y help -b b help -c c help + -z z help ''' @@ -4989,9 +4998,9 @@ def test_mutex_groups_with_mixed_optionals_positionals_wrap(self): g.add_argument('positional', nargs='?') usage = textwrap.dedent('''\ - usage: PROG [-h] [-v | -q | -x [EXTRA_LONG_OPTION_NAME] | - -y [YET_ANOTHER_LONG_OPTION] | - positional] + usage: PROG [-h] + [-v | -q | -x [EXTRA_LONG_OPTION_NAME] | + -y [YET_ANOTHER_LONG_OPTION] | positional] ''') self.assertEqual(parser.format_usage(), usage) diff --git a/Misc/NEWS.d/next/Library/2025-12-07-17-30-05.gh-issue-142346.okcAAp.rst b/Misc/NEWS.d/next/Library/2025-12-07-17-30-05.gh-issue-142346.okcAAp.rst new file mode 100644 index 00000000000000..cf570f314c00cc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-07-17-30-05.gh-issue-142346.okcAAp.rst @@ -0,0 +1,3 @@ +Fix usage formatting for mutually exclusive groups in :mod:`argparse` +when they are preceded by positional arguments or followed or intermixed +with other optional arguments. From f193c8fe9e1d722c9a7f9a2b15f8f1b913755491 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Sun, 7 Dec 2025 20:01:01 +0000 Subject: [PATCH 292/638] gh-141794: Reduce size of compiler stress tests to fix Android warnings (#142263) --- Lib/test/test_ast/test_ast.py | 3 ++- Lib/test/test_compile.py | 11 +++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 608ffdfad1209a..d2b76b46dbe2eb 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -992,7 +992,8 @@ def next(self): @skip_wasi_stack_overflow() @skip_emscripten_stack_overflow() def test_ast_recursion_limit(self): - crash_depth = 500_000 + # Android test devices have less memory. + crash_depth = 100_000 if sys.platform == "android" else 500_000 success_depth = 200 if _testinternalcapi is not None: remaining = _testinternalcapi.get_c_recursion_remaining() diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 30f21875b22ab3..fa611f480d60fd 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -728,7 +728,8 @@ def test_yet_more_evil_still_undecodable(self): def test_compiler_recursion_limit(self): # Compiler frames are small limit = 100 - crash_depth = limit * 5000 + # Android test devices have less memory. + crash_depth = limit * (1000 if sys.platform == "android" else 5000) success_depth = limit def check_limit(prefix, repeated, mode="single"): @@ -1036,11 +1037,13 @@ def test_path_like_objects(self): # An implicit test for PyUnicode_FSDecoder(). compile("42", FakePath("test_compile_pathlike"), "single") + # bpo-31113: Stack overflow when compile a long sequence of + # complex statements. @support.requires_resource('cpu') def test_stack_overflow(self): - # bpo-31113: Stack overflow when compile a long sequence of - # complex statements. - compile("if a: b\n" * 200000, "", "exec") + # Android test devices have less memory. + size = 100_000 if sys.platform == "android" else 200_000 + compile("if a: b\n" * size, "", "exec") # Multiple users rely on the fact that CPython does not generate # bytecode for dead code blocks. See bpo-37500 for more context. From dc9f2385ed528caf4ab02965b6d3e16b8a72db25 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Sun, 7 Dec 2025 13:02:12 -0800 Subject: [PATCH 293/638] GH-139862: Fix direct instantiation of `HelpFormatter` (#142384) --- Lib/argparse.py | 2 ++ Lib/test/test_argparse.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/Lib/argparse.py b/Lib/argparse.py index 6ed7386d0dd407..398825508f5917 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -189,6 +189,8 @@ def __init__( self._whitespace_matcher = _re.compile(r'\s+', _re.ASCII) self._long_break_matcher = _re.compile(r'\n\n\n+') + self._set_color(False) + def _set_color(self, color): from _colorize import can_colorize, decolor, get_theme diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index fe5232e9f3b591..7c5eed21219de0 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -5694,6 +5694,11 @@ def custom_formatter(prog): a-very-long-command command that does something ''')) + def test_direct_formatter_instantiation(self): + formatter = argparse.HelpFormatter(prog="program") + formatter.add_usage(usage=None, actions=[], groups=[]) + help_text = formatter.format_help() + self.assertEqual(help_text, "usage: program\n") # ===================================== # Optional/Positional constructor tests From ff2577f56eb2170ef0afafa90f78c693df7ca562 Mon Sep 17 00:00:00 2001 From: dr-carlos <77367421+dr-carlos@users.noreply.github.com> Date: Mon, 8 Dec 2025 07:34:04 +1030 Subject: [PATCH 294/638] gh-141732: Fix `ExceptionGroup` repr changing when original exception sequence is mutated (#141736) --- Doc/library/exceptions.rst | 6 ++ Include/cpython/pyerrors.h | 1 + Lib/test/test_exception_group.py | 73 +++++++++++++++- ...-11-19-16-40-24.gh-issue-141732.PTetqp.rst | 2 + Objects/exceptions.c | 87 ++++++++++++++++--- 5 files changed, 157 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-19-16-40-24.gh-issue-141732.PTetqp.rst diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index 16d42c010f6df0..b5e3a84b4556dd 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -978,6 +978,12 @@ their subgroups based on the types of the contained exceptions. raises a :exc:`TypeError` if any contained exception is not an :exc:`Exception` subclass. + .. impl-detail:: + + The ``excs`` parameter may be any sequence, but lists and tuples are + specifically processed more efficiently here. For optimal performance, + pass a tuple as ``excs``. + .. attribute:: message The ``msg`` argument to the constructor. This is a read-only attribute. diff --git a/Include/cpython/pyerrors.h b/Include/cpython/pyerrors.h index 6b63d304b0d929..be2e3b641c25cb 100644 --- a/Include/cpython/pyerrors.h +++ b/Include/cpython/pyerrors.h @@ -18,6 +18,7 @@ typedef struct { PyException_HEAD PyObject *msg; PyObject *excs; + PyObject *excs_str; } PyBaseExceptionGroupObject; typedef struct { diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index 5df2c41c6b56bc..ace7ec72917934 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -1,4 +1,4 @@ -import collections.abc +import collections import types import unittest from test.support import skip_emscripten_stack_overflow, skip_wasi_stack_overflow, exceeds_recursion_limit @@ -193,6 +193,77 @@ class MyEG(ExceptionGroup): "MyEG('flat', [ValueError(1), TypeError(2)]), " "TypeError(2)])")) + def test_exceptions_mutation(self): + class MyEG(ExceptionGroup): + pass + + excs = [ValueError(1), TypeError(2)] + eg = MyEG('test', excs) + + self.assertEqual(repr(eg), "MyEG('test', [ValueError(1), TypeError(2)])") + excs.clear() + + # Ensure that clearing the exceptions sequence doesn't change the repr. + self.assertEqual(repr(eg), "MyEG('test', [ValueError(1), TypeError(2)])") + + # Ensure that the args are still as passed. + self.assertEqual(eg.args, ('test', [])) + + excs = (ValueError(1), KeyboardInterrupt(2)) + eg = BaseExceptionGroup('test', excs) + + # Ensure that immutable sequences still work fine. + self.assertEqual( + repr(eg), + "BaseExceptionGroup('test', (ValueError(1), KeyboardInterrupt(2)))" + ) + + # Test non-standard custom sequences. + excs = collections.deque([ValueError(1), TypeError(2)]) + eg = ExceptionGroup('test', excs) + + self.assertEqual( + repr(eg), + "ExceptionGroup('test', deque([ValueError(1), TypeError(2)]))" + ) + excs.clear() + + # Ensure that clearing the exceptions sequence doesn't change the repr. + self.assertEqual( + repr(eg), + "ExceptionGroup('test', deque([ValueError(1), TypeError(2)]))" + ) + + def test_repr_raises(self): + class MySeq(collections.abc.Sequence): + def __init__(self, raises): + self.raises = raises + + def __len__(self): + return 1 + + def __getitem__(self, index): + if index == 0: + return ValueError(1) + raise IndexError + + def __repr__(self): + if self.raises: + raise self.raises + return None + + seq = MySeq(None) + with self.assertRaisesRegex( + TypeError, + r".*MySeq\.__repr__\(\) must return a str, not NoneType" + ): + ExceptionGroup("test", seq) + + seq = MySeq(ValueError) + with self.assertRaises(ValueError): + BaseExceptionGroup("test", seq) + + def create_simple_eg(): excs = [] diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-19-16-40-24.gh-issue-141732.PTetqp.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-19-16-40-24.gh-issue-141732.PTetqp.rst new file mode 100644 index 00000000000000..08420fd5f4d18a --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-19-16-40-24.gh-issue-141732.PTetqp.rst @@ -0,0 +1,2 @@ +Ensure the :meth:`~object.__repr__` for :exc:`ExceptionGroup` and :exc:`BaseExceptionGroup` does +not change when the exception sequence that was original passed in to its constructor is subsequently mutated. diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 244d8f39e2bae5..9a43057b383d29 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -694,12 +694,12 @@ PyTypeObject _PyExc_ ## EXCNAME = { \ #define ComplexExtendsException(EXCBASE, EXCNAME, EXCSTORE, EXCNEW, \ EXCMETHODS, EXCMEMBERS, EXCGETSET, \ - EXCSTR, EXCDOC) \ + EXCSTR, EXCREPR, EXCDOC) \ static PyTypeObject _PyExc_ ## EXCNAME = { \ PyVarObject_HEAD_INIT(NULL, 0) \ # EXCNAME, \ sizeof(Py ## EXCSTORE ## Object), 0, \ - EXCSTORE ## _dealloc, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ + EXCSTORE ## _dealloc, 0, 0, 0, 0, EXCREPR, 0, 0, 0, 0, 0, \ EXCSTR, 0, 0, 0, \ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, \ PyDoc_STR(EXCDOC), EXCSTORE ## _traverse, \ @@ -792,7 +792,7 @@ StopIteration_traverse(PyObject *op, visitproc visit, void *arg) } ComplexExtendsException(PyExc_Exception, StopIteration, StopIteration, - 0, 0, StopIteration_members, 0, 0, + 0, 0, StopIteration_members, 0, 0, 0, "Signal the end from iterator.__next__()."); @@ -865,7 +865,7 @@ static PyMemberDef SystemExit_members[] = { }; ComplexExtendsException(PyExc_BaseException, SystemExit, SystemExit, - 0, 0, SystemExit_members, 0, 0, + 0, 0, SystemExit_members, 0, 0, 0, "Request to exit from the interpreter."); /* @@ -890,6 +890,7 @@ BaseExceptionGroup_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PyObject *message = NULL; PyObject *exceptions = NULL; + PyObject *exceptions_str = NULL; if (!PyArg_ParseTuple(args, "UO:BaseExceptionGroup.__new__", @@ -905,6 +906,18 @@ BaseExceptionGroup_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } + /* Save initial exceptions sequence as a string in case sequence is mutated */ + if (!PyList_Check(exceptions) && !PyTuple_Check(exceptions)) { + exceptions_str = PyObject_Repr(exceptions); + if (exceptions_str == NULL) { + /* We don't hold a reference to exceptions, so clear it before + * attempting a decref in the cleanup. + */ + exceptions = NULL; + goto error; + } + } + exceptions = PySequence_Tuple(exceptions); if (!exceptions) { return NULL; @@ -988,9 +1001,11 @@ BaseExceptionGroup_new(PyTypeObject *type, PyObject *args, PyObject *kwds) self->msg = Py_NewRef(message); self->excs = exceptions; + self->excs_str = exceptions_str; return (PyObject*)self; error: - Py_DECREF(exceptions); + Py_XDECREF(exceptions); + Py_XDECREF(exceptions_str); return NULL; } @@ -1029,6 +1044,7 @@ BaseExceptionGroup_clear(PyObject *op) PyBaseExceptionGroupObject *self = PyBaseExceptionGroupObject_CAST(op); Py_CLEAR(self->msg); Py_CLEAR(self->excs); + Py_CLEAR(self->excs_str); return BaseException_clear(op); } @@ -1046,6 +1062,7 @@ BaseExceptionGroup_traverse(PyObject *op, visitproc visit, void *arg) PyBaseExceptionGroupObject *self = PyBaseExceptionGroupObject_CAST(op); Py_VISIT(self->msg); Py_VISIT(self->excs); + Py_VISIT(self->excs_str); return BaseException_traverse(op, visit, arg); } @@ -1063,6 +1080,54 @@ BaseExceptionGroup_str(PyObject *op) self->msg, num_excs, num_excs > 1 ? "s" : ""); } +static PyObject * +BaseExceptionGroup_repr(PyObject *op) +{ + PyBaseExceptionGroupObject *self = PyBaseExceptionGroupObject_CAST(op); + assert(self->msg); + + PyObject *exceptions_str = NULL; + + /* Use the saved exceptions string for custom sequences. */ + if (self->excs_str) { + exceptions_str = Py_NewRef(self->excs_str); + } + else { + assert(self->excs); + + /* Older versions delegated to BaseException, inserting the current + * value of self.args[1]; but this can be mutable and go out-of-sync + * with self.exceptions. Instead, use self.exceptions for accuracy, + * making it look like self.args[1] for backwards compatibility. */ + if (PyList_Check(PyTuple_GET_ITEM(self->args, 1))) { + PyObject *exceptions_list = PySequence_List(self->excs); + if (!exceptions_list) { + return NULL; + } + + exceptions_str = PyObject_Repr(exceptions_list); + Py_DECREF(exceptions_list); + } + else { + exceptions_str = PyObject_Repr(self->excs); + } + + if (!exceptions_str) { + return NULL; + } + } + + assert(exceptions_str != NULL); + + const char *name = _PyType_Name(Py_TYPE(self)); + PyObject *repr = PyUnicode_FromFormat( + "%s(%R, %U)", name, + self->msg, exceptions_str); + + Py_DECREF(exceptions_str); + return repr; +} + /*[clinic input] @critical_section BaseExceptionGroup.derive @@ -1697,7 +1762,7 @@ static PyMethodDef BaseExceptionGroup_methods[] = { ComplexExtendsException(PyExc_BaseException, BaseExceptionGroup, BaseExceptionGroup, BaseExceptionGroup_new /* new */, BaseExceptionGroup_methods, BaseExceptionGroup_members, - 0 /* getset */, BaseExceptionGroup_str, + 0 /* getset */, BaseExceptionGroup_str, BaseExceptionGroup_repr, "A combination of multiple unrelated exceptions."); /* @@ -2425,7 +2490,7 @@ static PyGetSetDef OSError_getset[] = { ComplexExtendsException(PyExc_Exception, OSError, OSError, OSError_new, OSError_methods, OSError_members, OSError_getset, - OSError_str, + OSError_str, 0, "Base class for I/O related errors."); @@ -2566,7 +2631,7 @@ static PyMethodDef NameError_methods[] = { ComplexExtendsException(PyExc_Exception, NameError, NameError, 0, NameError_methods, NameError_members, - 0, BaseException_str, "Name not found globally."); + 0, BaseException_str, 0, "Name not found globally."); /* * UnboundLocalError extends NameError @@ -2700,7 +2765,7 @@ static PyMethodDef AttributeError_methods[] = { ComplexExtendsException(PyExc_Exception, AttributeError, AttributeError, 0, AttributeError_methods, AttributeError_members, - 0, BaseException_str, "Attribute not found."); + 0, BaseException_str, 0, "Attribute not found."); /* * SyntaxError extends Exception @@ -2899,7 +2964,7 @@ static PyMemberDef SyntaxError_members[] = { ComplexExtendsException(PyExc_Exception, SyntaxError, SyntaxError, 0, 0, SyntaxError_members, 0, - SyntaxError_str, "Invalid syntax."); + SyntaxError_str, 0, "Invalid syntax."); /* @@ -2959,7 +3024,7 @@ KeyError_str(PyObject *op) } ComplexExtendsException(PyExc_LookupError, KeyError, BaseException, - 0, 0, 0, 0, KeyError_str, "Mapping key not found."); + 0, 0, 0, 0, KeyError_str, 0, "Mapping key not found."); /* From ef51a7c8f3f4511ad18ee310798883d15bc6d9b7 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 7 Dec 2025 22:41:15 +0000 Subject: [PATCH 295/638] gh-138122: Make sampling profiler integration tests more resilient (#142382) The tests were flaky on slow machines because subprocesses could finish before enough samples were collected. This adds synchronization similar to test_external_inspection: test scripts now signal when they start working, and the profiler waits for this signal before sampling. Test scripts now run in infinite loops until killed rather than for fixed iterations, ensuring the profiler always has active work to sample regardless of machine speed. --- .../test_sampling_profiler/helpers.py | 99 +++++++++++++-- .../test_sampling_profiler/test_advanced.py | 36 ++---- .../test_integration.py | 119 ++++++++++-------- .../test_sampling_profiler/test_modes.py | 57 +++------ 4 files changed, 185 insertions(+), 126 deletions(-) diff --git a/Lib/test/test_profiling/test_sampling_profiler/helpers.py b/Lib/test/test_profiling/test_sampling_profiler/helpers.py index f1c01afd0fa555..0e32d8dd9eabef 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/helpers.py +++ b/Lib/test/test_profiling/test_sampling_profiler/helpers.py @@ -38,12 +38,88 @@ SubprocessInfo = namedtuple("SubprocessInfo", ["process", "socket"]) +def _wait_for_signal(sock, expected_signals, timeout=SHORT_TIMEOUT): + """ + Wait for expected signal(s) from a socket with proper timeout and EOF handling. + + Args: + sock: Connected socket to read from + expected_signals: Single bytes object or list of bytes objects to wait for + timeout: Socket timeout in seconds + + Returns: + bytes: Complete accumulated response buffer + + Raises: + RuntimeError: If connection closed before signal received or timeout + """ + if isinstance(expected_signals, bytes): + expected_signals = [expected_signals] + + sock.settimeout(timeout) + buffer = b"" + + while True: + # Check if all expected signals are in buffer + if all(sig in buffer for sig in expected_signals): + return buffer + + try: + chunk = sock.recv(4096) + if not chunk: + raise RuntimeError( + f"Connection closed before receiving expected signals. " + f"Expected: {expected_signals}, Got: {buffer[-200:]!r}" + ) + buffer += chunk + except socket.timeout: + raise RuntimeError( + f"Timeout waiting for signals. " + f"Expected: {expected_signals}, Got: {buffer[-200:]!r}" + ) from None + except OSError as e: + raise RuntimeError( + f"Socket error while waiting for signals: {e}. " + f"Expected: {expected_signals}, Got: {buffer[-200:]!r}" + ) from None + + +def _cleanup_sockets(*sockets): + """Safely close multiple sockets, ignoring errors.""" + for sock in sockets: + if sock is not None: + try: + sock.close() + except OSError: + pass + + +def _cleanup_process(proc, timeout=SHORT_TIMEOUT): + """Terminate a process gracefully, escalating to kill if needed.""" + if proc.poll() is not None: + return + proc.terminate() + try: + proc.wait(timeout=timeout) + return + except subprocess.TimeoutExpired: + pass + proc.kill() + try: + proc.wait(timeout=timeout) + except subprocess.TimeoutExpired: + pass # Process refuses to die, nothing more we can do + + @contextlib.contextmanager -def test_subprocess(script): +def test_subprocess(script, wait_for_working=False): """Context manager to create a test subprocess with socket synchronization. Args: - script: Python code to execute in the subprocess + script: Python code to execute in the subprocess. If wait_for_working + is True, script should send b"working" after starting work. + wait_for_working: If True, wait for both "ready" and "working" signals. + Default False for backward compatibility. Yields: SubprocessInfo: Named tuple with process and socket objects @@ -80,19 +156,18 @@ def test_subprocess(script): # Wait for process to connect and send ready signal client_socket, _ = server_socket.accept() server_socket.close() - response = client_socket.recv(1024) - if response != b"ready": - raise RuntimeError( - f"Unexpected response from subprocess: {response!r}" - ) + server_socket = None + + # Wait for ready signal, and optionally working signal + if wait_for_working: + _wait_for_signal(client_socket, [b"ready", b"working"]) + else: + _wait_for_signal(client_socket, b"ready") yield SubprocessInfo(proc, client_socket) finally: - if client_socket is not None: - client_socket.close() - if proc.poll() is None: - proc.kill() - proc.wait() + _cleanup_sockets(client_socket, server_socket) + _cleanup_process(proc) def close_and_unlink(file): diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_advanced.py b/Lib/test/test_profiling/test_sampling_profiler/test_advanced.py index 94946d74aa4784..843fb3b7416375 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_advanced.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_advanced.py @@ -39,32 +39,26 @@ def setUpClass(cls): import gc class ExpensiveGarbage: - """Class that triggers GC with expensive finalizer (callback).""" def __init__(self): self.cycle = self def __del__(self): - # CPU-intensive work in the finalizer callback result = 0 for i in range(100000): result += i * i if i % 1000 == 0: result = result % 1000000 -def main_loop(): - """Main loop that triggers GC with expensive callback.""" - while True: - ExpensiveGarbage() - gc.collect() - -if __name__ == "__main__": - main_loop() +_test_sock.sendall(b"working") +while True: + ExpensiveGarbage() + gc.collect() ''' def test_gc_frames_enabled(self): """Test that GC frames appear when gc tracking is enabled.""" with ( - test_subprocess(self.gc_test_script) as subproc, + test_subprocess(self.gc_test_script, wait_for_working=True) as subproc, io.StringIO() as captured_output, mock.patch("sys.stdout", captured_output), ): @@ -94,7 +88,7 @@ def test_gc_frames_enabled(self): def test_gc_frames_disabled(self): """Test that GC frames do not appear when gc tracking is disabled.""" with ( - test_subprocess(self.gc_test_script) as subproc, + test_subprocess(self.gc_test_script, wait_for_working=True) as subproc, io.StringIO() as captured_output, mock.patch("sys.stdout", captured_output), ): @@ -133,18 +127,13 @@ def setUpClass(cls): cls.native_test_script = """ import operator -def main_loop(): - while True: - # Native code in the middle of the stack: - operator.call(inner) - def inner(): - # Python code at the top of the stack: for _ in range(1_000_0000): pass -if __name__ == "__main__": - main_loop() +_test_sock.sendall(b"working") +while True: + operator.call(inner) """ def test_native_frames_enabled(self): @@ -154,10 +143,7 @@ def test_native_frames_enabled(self): ) self.addCleanup(close_and_unlink, collapsed_file) - with ( - test_subprocess(self.native_test_script) as subproc, - ): - # Suppress profiler output when testing file export + with test_subprocess(self.native_test_script, wait_for_working=True) as subproc: with ( io.StringIO() as captured_output, mock.patch("sys.stdout", captured_output), @@ -199,7 +185,7 @@ def test_native_frames_enabled(self): def test_native_frames_disabled(self): """Test that native frames do not appear when native tracking is disabled.""" with ( - test_subprocess(self.native_test_script) as subproc, + test_subprocess(self.native_test_script, wait_for_working=True) as subproc, io.StringIO() as captured_output, mock.patch("sys.stdout", captured_output), ): diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_integration.py b/Lib/test/test_profiling/test_sampling_profiler/test_integration.py index aae241a3335e37..e92b3f45fbc379 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_integration.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_integration.py @@ -39,6 +39,9 @@ # Duration for profiling tests - long enough for process to complete naturally PROFILING_TIMEOUT = str(int(SHORT_TIMEOUT)) +# Duration for profiling in tests - short enough to complete quickly +PROFILING_DURATION_SEC = 2 + @skip_if_not_supported @unittest.skipIf( @@ -359,23 +362,14 @@ def total_occurrences(func): self.assertEqual(total_occurrences(main_key), 2) -@requires_subprocess() -@skip_if_not_supported -class TestSampleProfilerIntegration(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.test_script = ''' -import time -import os - +# Shared workload functions for test scripts +_WORKLOAD_FUNCTIONS = ''' def slow_fibonacci(n): - """Recursive fibonacci - should show up prominently in profiler.""" if n <= 1: return n return slow_fibonacci(n-1) + slow_fibonacci(n-2) def cpu_intensive_work(): - """CPU intensive work that should show in profiler.""" result = 0 for i in range(10000): result += i * i @@ -383,33 +377,48 @@ def cpu_intensive_work(): result = result % 1000000 return result -def main_loop(): - """Main test loop.""" - max_iterations = 200 - - for iteration in range(max_iterations): +def do_work(): + iteration = 0 + while True: if iteration % 2 == 0: - result = slow_fibonacci(15) + slow_fibonacci(15) else: - result = cpu_intensive_work() + cpu_intensive_work() + iteration += 1 +''' + -if __name__ == "__main__": - main_loop() +@requires_subprocess() +@skip_if_not_supported +class TestSampleProfilerIntegration(unittest.TestCase): + @classmethod + def setUpClass(cls): + # Test script for use with test_subprocess() - signals when work starts + cls.test_script = _WORKLOAD_FUNCTIONS + ''' +_test_sock.sendall(b"working") +do_work() +''' + # CLI test script - runs for fixed duration (no socket sync) + cls.cli_test_script = ''' +import time +''' + _WORKLOAD_FUNCTIONS.replace( + 'while True:', 'end_time = time.time() + 30\n while time.time() < end_time:' +) + ''' +do_work() ''' def test_sampling_basic_functionality(self): with ( - test_subprocess(self.test_script) as subproc, + test_subprocess(self.test_script, wait_for_working=True) as subproc, io.StringIO() as captured_output, mock.patch("sys.stdout", captured_output), ): try: - # Sample for up to SHORT_TIMEOUT seconds, but process exits after fixed iterations collector = PstatsCollector(sample_interval_usec=1000, skip_idle=False) profiling.sampling.sample.sample( subproc.process.pid, collector, - duration_sec=SHORT_TIMEOUT, + duration_sec=PROFILING_DURATION_SEC, ) collector.print_stats(show_summary=False) except PermissionError: @@ -431,7 +440,7 @@ def test_sampling_with_pstats_export(self): ) self.addCleanup(close_and_unlink, pstats_out) - with test_subprocess(self.test_script) as subproc: + with test_subprocess(self.test_script, wait_for_working=True) as subproc: # Suppress profiler output when testing file export with ( io.StringIO() as captured_output, @@ -442,7 +451,7 @@ def test_sampling_with_pstats_export(self): profiling.sampling.sample.sample( subproc.process.pid, collector, - duration_sec=1, + duration_sec=PROFILING_DURATION_SEC, ) collector.export(pstats_out.name) except PermissionError: @@ -476,7 +485,7 @@ def test_sampling_with_collapsed_export(self): self.addCleanup(close_and_unlink, collapsed_file) with ( - test_subprocess(self.test_script) as subproc, + test_subprocess(self.test_script, wait_for_working=True) as subproc, ): # Suppress profiler output when testing file export with ( @@ -488,7 +497,7 @@ def test_sampling_with_collapsed_export(self): profiling.sampling.sample.sample( subproc.process.pid, collector, - duration_sec=1, + duration_sec=PROFILING_DURATION_SEC, ) collector.export(collapsed_file.name) except PermissionError: @@ -526,7 +535,7 @@ def test_sampling_with_collapsed_export(self): def test_sampling_all_threads(self): with ( - test_subprocess(self.test_script) as subproc, + test_subprocess(self.test_script, wait_for_working=True) as subproc, # Suppress profiler output io.StringIO() as captured_output, mock.patch("sys.stdout", captured_output), @@ -536,7 +545,7 @@ def test_sampling_all_threads(self): profiling.sampling.sample.sample( subproc.process.pid, collector, - duration_sec=1, + duration_sec=PROFILING_DURATION_SEC, all_threads=True, ) collector.print_stats(show_summary=False) @@ -548,12 +557,16 @@ def test_sampling_all_threads(self): def test_sample_target_script(self): script_file = tempfile.NamedTemporaryFile(delete=False) - script_file.write(self.test_script.encode("utf-8")) + script_file.write(self.cli_test_script.encode("utf-8")) script_file.flush() self.addCleanup(close_and_unlink, script_file) - # Sample for up to SHORT_TIMEOUT seconds, but process exits after fixed iterations - test_args = ["profiling.sampling.sample", "run", "-d", PROFILING_TIMEOUT, script_file.name] + # Sample for PROFILING_DURATION_SEC seconds + test_args = [ + "profiling.sampling.sample", "run", + "-d", str(PROFILING_DURATION_SEC), + script_file.name + ] with ( mock.patch("sys.argv", test_args), @@ -583,13 +596,13 @@ def test_sample_target_module(self): module_path = os.path.join(tempdir.name, "test_module.py") with open(module_path, "w") as f: - f.write(self.test_script) + f.write(self.cli_test_script) test_args = [ "profiling.sampling.cli", "run", "-d", - PROFILING_TIMEOUT, + str(PROFILING_DURATION_SEC), "-m", "test_module", ] @@ -630,8 +643,10 @@ def test_invalid_pid(self): profiling.sampling.sample.sample(-1, collector, duration_sec=1) def test_process_dies_during_sampling(self): + # Use wait_for_working=False since this simple script doesn't send "working" with test_subprocess( - "import time; time.sleep(0.5); exit()" + "import time; time.sleep(0.5); exit()", + wait_for_working=False ) as subproc: with ( io.StringIO() as captured_output, @@ -654,7 +669,11 @@ def test_process_dies_during_sampling(self): self.assertIn("Error rate", output) def test_is_process_running(self): - with test_subprocess("import time; time.sleep(1000)") as subproc: + # Use wait_for_working=False since this simple script doesn't send "working" + with test_subprocess( + "import time; time.sleep(1000)", + wait_for_working=False + ) as subproc: try: profiler = SampleProfiler( pid=subproc.process.pid, @@ -681,7 +700,11 @@ def test_is_process_running(self): @unittest.skipUnless(sys.platform == "linux", "Only valid on Linux") def test_esrch_signal_handling(self): - with test_subprocess("import time; time.sleep(1000)") as subproc: + # Use wait_for_working=False since this simple script doesn't send "working" + with test_subprocess( + "import time; time.sleep(1000)", + wait_for_working=False + ) as subproc: try: unwinder = _remote_debugging.RemoteUnwinder( subproc.process.pid @@ -793,38 +816,34 @@ class TestAsyncAwareProfilingIntegration(unittest.TestCase): @classmethod def setUpClass(cls): + # Async test script that runs indefinitely until killed. + # Sends "working" signal AFTER tasks are created and scheduled. cls.async_script = ''' import asyncio async def sleeping_leaf(): - """Leaf task that just sleeps - visible in 'all' mode.""" - for _ in range(50): + while True: await asyncio.sleep(0.02) async def cpu_leaf(): - """Leaf task that does CPU work - visible in both modes.""" total = 0 - for _ in range(200): + while True: for i in range(10000): total += i * i await asyncio.sleep(0) - return total async def supervisor(): - """Middle layer that spawns leaf tasks.""" tasks = [ asyncio.create_task(sleeping_leaf(), name="Sleeper-0"), asyncio.create_task(sleeping_leaf(), name="Sleeper-1"), asyncio.create_task(sleeping_leaf(), name="Sleeper-2"), asyncio.create_task(cpu_leaf(), name="Worker"), ] + await asyncio.sleep(0) # Let tasks get scheduled + _test_sock.sendall(b"working") await asyncio.gather(*tasks) -async def main(): - await supervisor() - -if __name__ == "__main__": - asyncio.run(main()) +asyncio.run(supervisor()) ''' def _collect_async_samples(self, async_aware_mode): @@ -832,13 +851,13 @@ def _collect_async_samples(self, async_aware_mode): Returns a dict mapping function names to their sample counts. """ - with test_subprocess(self.async_script) as subproc: + with test_subprocess(self.async_script, wait_for_working=True) as subproc: try: collector = CollapsedStackCollector(1000, skip_idle=False) profiling.sampling.sample.sample( subproc.process.pid, collector, - duration_sec=SHORT_TIMEOUT, + duration_sec=PROFILING_DURATION_SEC, async_aware=async_aware_mode, ) except PermissionError: diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_modes.py b/Lib/test/test_profiling/test_sampling_profiler/test_modes.py index 1b0e21a5fe45d6..c0457ee7eb8357 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_modes.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_modes.py @@ -143,27 +143,16 @@ def cpu_active_worker(): while True: x += 1 -def main(): - # Start both threads - idle_thread = threading.Thread(target=idle_worker) - cpu_thread = threading.Thread(target=cpu_active_worker) - idle_thread.start() - cpu_thread.start() - - # Wait for CPU thread to be running, then signal test - cpu_ready.wait() - _test_sock.sendall(b"threads_ready") - - idle_thread.join() - cpu_thread.join() - -main() - +idle_thread = threading.Thread(target=idle_worker) +cpu_thread = threading.Thread(target=cpu_active_worker) +idle_thread.start() +cpu_thread.start() +cpu_ready.wait() +_test_sock.sendall(b"working") +idle_thread.join() +cpu_thread.join() """ - with test_subprocess(cpu_vs_idle_script) as subproc: - # Wait for signal that threads are running - response = subproc.socket.recv(1024) - self.assertEqual(response, b"threads_ready") + with test_subprocess(cpu_vs_idle_script, wait_for_working=True) as subproc: with ( io.StringIO() as captured_output, @@ -365,26 +354,16 @@ def gil_holding_work(): while True: x += 1 -def main(): - # Start both threads - idle_thread = threading.Thread(target=gil_releasing_work) - cpu_thread = threading.Thread(target=gil_holding_work) - idle_thread.start() - cpu_thread.start() - - # Wait for GIL-holding thread to be running, then signal test - gil_ready.wait() - _test_sock.sendall(b"threads_ready") - - idle_thread.join() - cpu_thread.join() - -main() +idle_thread = threading.Thread(target=gil_releasing_work) +cpu_thread = threading.Thread(target=gil_holding_work) +idle_thread.start() +cpu_thread.start() +gil_ready.wait() +_test_sock.sendall(b"working") +idle_thread.join() +cpu_thread.join() """ - with test_subprocess(gil_test_script) as subproc: - # Wait for signal that threads are running - response = subproc.socket.recv(1024) - self.assertEqual(response, b"threads_ready") + with test_subprocess(gil_test_script, wait_for_working=True) as subproc: with ( io.StringIO() as captured_output, From 3fa1425bfbab1cb7a4e0bd9c92483f11dccd1181 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Mon, 8 Dec 2025 02:51:51 +0200 Subject: [PATCH 296/638] gh-142363: Improve Tachyon flamegraph contrast (#142377) --- Lib/profiling/sampling/_flamegraph_assets/flamegraph.css | 2 +- Lib/profiling/sampling/_shared_assets/base.css | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/profiling/sampling/_flamegraph_assets/flamegraph.css b/Lib/profiling/sampling/_flamegraph_assets/flamegraph.css index c75f2324b6d499..18d2279da9b645 100644 --- a/Lib/profiling/sampling/_flamegraph_assets/flamegraph.css +++ b/Lib/profiling/sampling/_flamegraph_assets/flamegraph.css @@ -789,7 +789,7 @@ body.resizing-sidebar { .tooltip-location { font-family: var(--font-mono); font-size: 11px; - color: var(--text-muted); + color: var(--text-secondary); background: var(--bg-tertiary); padding: 4px 8px; border-radius: 4px; diff --git a/Lib/profiling/sampling/_shared_assets/base.css b/Lib/profiling/sampling/_shared_assets/base.css index d9223a98c0f756..54e3d78f8ebf34 100644 --- a/Lib/profiling/sampling/_shared_assets/base.css +++ b/Lib/profiling/sampling/_shared_assets/base.css @@ -44,7 +44,7 @@ --text-primary: #2e3338; --text-secondary: #5a6c7d; - --text-muted: #8b949e; + --text-muted: #6f767e; --accent: #3776ab; --accent-hover: #2d5aa0; @@ -91,7 +91,7 @@ --text-primary: #e6edf3; --text-secondary: #8b949e; - --text-muted: #6e7681; + --text-muted: #757e8a; --accent: #58a6ff; --accent-hover: #79b8ff; From 7099af8f5e6966bc0179b74c8306506d892282e7 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Mon, 8 Dec 2025 12:08:06 +0800 Subject: [PATCH 297/638] gh-139946: distinguish stdout or stderr when colorizing output in argparse (#140495) Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Savannah Ostrowski --- Lib/argparse.py | 44 +++++++++++++------ Lib/test/test_argparse.py | 34 ++++++++++++++ ...-10-23-06-38-35.gh-issue-139946.HZa5hu.rst | 1 + 3 files changed, 65 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-10-23-06-38-35.gh-issue-139946.HZa5hu.rst diff --git a/Lib/argparse.py b/Lib/argparse.py index 398825508f5917..1d550264ae420f 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -89,8 +89,8 @@ import os as _os import re as _re import sys as _sys - -from gettext import gettext as _, ngettext +from gettext import gettext as _ +from gettext import ngettext SUPPRESS = '==SUPPRESS==' @@ -191,10 +191,10 @@ def __init__( self._set_color(False) - def _set_color(self, color): + def _set_color(self, color, *, file=None): from _colorize import can_colorize, decolor, get_theme - if color and can_colorize(): + if color and can_colorize(file=file): self._theme = get_theme(force_color=True).argparse self._decolor = decolor else: @@ -1675,7 +1675,7 @@ def _get_optional_kwargs(self, *args, **kwargs): option_strings = [] for option_string in args: # error on strings that don't start with an appropriate prefix - if not option_string[0] in self.prefix_chars: + if option_string[0] not in self.prefix_chars: raise ValueError( f'invalid option string {option_string!r}: ' f'must start with a character {self.prefix_chars!r}') @@ -2455,7 +2455,7 @@ def _parse_optional(self, arg_string): return None # if it doesn't start with a prefix, it was meant to be positional - if not arg_string[0] in self.prefix_chars: + if arg_string[0] not in self.prefix_chars: return None # if the option string is present in the parser, return the action @@ -2717,14 +2717,16 @@ def _check_value(self, action, value): # Help-formatting methods # ======================= - def format_usage(self): - formatter = self._get_formatter() + def format_usage(self, formatter=None): + if formatter is None: + formatter = self._get_formatter() formatter.add_usage(self.usage, self._actions, self._mutually_exclusive_groups) return formatter.format_help() - def format_help(self): - formatter = self._get_formatter() + def format_help(self, formatter=None): + if formatter is None: + formatter = self._get_formatter() # usage formatter.add_usage(self.usage, self._actions, @@ -2746,9 +2748,9 @@ def format_help(self): # determine help from format above return formatter.format_help() - def _get_formatter(self): + def _get_formatter(self, file=None): formatter = self.formatter_class(prog=self.prog) - formatter._set_color(self.color) + formatter._set_color(self.color, file=file) return formatter def _get_validation_formatter(self): @@ -2765,12 +2767,26 @@ def _get_validation_formatter(self): def print_usage(self, file=None): if file is None: file = _sys.stdout - self._print_message(self.format_usage(), file) + formatter = self._get_formatter(file=file) + try: + usage_text = self.format_usage(formatter=formatter) + except TypeError: + # Backward compatibility for formatter classes that + # do not accept the 'formatter' keyword argument. + usage_text = self.format_usage() + self._print_message(usage_text, file) def print_help(self, file=None): if file is None: file = _sys.stdout - self._print_message(self.format_help(), file) + formatter = self._get_formatter(file=file) + try: + help_text = self.format_help(formatter=formatter) + except TypeError: + # Backward compatibility for formatter classes that + # do not accept the 'formatter' keyword argument. + help_text = self.format_help() + self._print_message(help_text, file) def _print_message(self, message, file=None): if message: diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 7c5eed21219de0..ab5382e41e7871 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -7558,6 +7558,40 @@ def test_error_and_warning_not_colorized_when_disabled(self): self.assertNotIn('\x1b[', warn) self.assertIn('warning:', warn) + def test_print_help_uses_target_file_for_color_decision(self): + parser = argparse.ArgumentParser(prog='PROG', color=True) + parser.add_argument('--opt') + output = io.StringIO() + calls = [] + + def fake_can_colorize(*, file=None): + calls.append(file) + return file is None + + with swap_attr(_colorize, 'can_colorize', fake_can_colorize): + parser.print_help(file=output) + + self.assertIs(calls[-1], output) + self.assertIn(output, calls) + self.assertNotIn('\x1b[', output.getvalue()) + + def test_print_usage_uses_target_file_for_color_decision(self): + parser = argparse.ArgumentParser(prog='PROG', color=True) + parser.add_argument('--opt') + output = io.StringIO() + calls = [] + + def fake_can_colorize(*, file=None): + calls.append(file) + return file is None + + with swap_attr(_colorize, 'can_colorize', fake_can_colorize): + parser.print_usage(file=output) + + self.assertIs(calls[-1], output) + self.assertIn(output, calls) + self.assertNotIn('\x1b[', output.getvalue()) + class TestModule(unittest.TestCase): def test_deprecated__version__(self): diff --git a/Misc/NEWS.d/next/Library/2025-10-23-06-38-35.gh-issue-139946.HZa5hu.rst b/Misc/NEWS.d/next/Library/2025-10-23-06-38-35.gh-issue-139946.HZa5hu.rst new file mode 100644 index 00000000000000..fb47931728414d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-23-06-38-35.gh-issue-139946.HZa5hu.rst @@ -0,0 +1 @@ +Distinguish stdout and stderr when colorizing output in argparse module. From 3db7bf2d180be5475266411b62114c28a0d4f92c Mon Sep 17 00:00:00 2001 From: yihong Date: Mon, 8 Dec 2025 12:45:04 +0800 Subject: [PATCH 298/638] gh-142207: remove assertions incompatible under `profiling.sampling` (#142331) --- .../Library/2025-12-06-13-19-43.gh-issue-142207.x_X9oH.rst | 2 ++ Modules/_remote_debugging/threads.c | 6 ++---- 2 files changed, 4 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-12-06-13-19-43.gh-issue-142207.x_X9oH.rst diff --git a/Misc/NEWS.d/next/Library/2025-12-06-13-19-43.gh-issue-142207.x_X9oH.rst b/Misc/NEWS.d/next/Library/2025-12-06-13-19-43.gh-issue-142207.x_X9oH.rst new file mode 100644 index 00000000000000..69ca8c41ac9b8b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-06-13-19-43.gh-issue-142207.x_X9oH.rst @@ -0,0 +1,2 @@ +Fix: profiling.sampling may cause assertion ``!(has_gil && +gil_requested)`` diff --git a/Modules/_remote_debugging/threads.c b/Modules/_remote_debugging/threads.c index 69819eb8dcd645..774338f9dc241e 100644 --- a/Modules/_remote_debugging/threads.c +++ b/Modules/_remote_debugging/threads.c @@ -339,12 +339,10 @@ unwind_stack_for_thread( #endif if (has_gil) { status_flags |= THREAD_STATUS_HAS_GIL; + // gh-142207 for remote debugging. + gil_requested = 0; } - // Assert that we never have both HAS_GIL and GIL_REQUESTED set at the same time - // This would indicate a race condition in the GIL state tracking - assert(!(has_gil && gil_requested)); - // Check CPU status long pthread_id = GET_MEMBER(long, ts, unwinder->debug_offsets.thread_state.thread_id); From 8620d3009ab2add053a4dfee65fdddc0ca0dfcc0 Mon Sep 17 00:00:00 2001 From: Apurva Khatri Date: Mon, 8 Dec 2025 04:54:59 -0500 Subject: [PATCH 299/638] gh-108202: ``calendar``: Document ``prweek`` (#108466) Co-authored-by: apurvakhatri Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/library/calendar.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index f76b1013dfbc66..822e627af8db95 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -158,6 +158,11 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is :class:`TextCalendar` instances have the following methods: + .. method:: prweek(theweek, width) + + Print a week's calendar as returned by :meth:`formatweek` and without a + final newline. + .. method:: formatday(theday, weekday, width) From c279e95367922fd0287c2212bb5ec6452d73353e Mon Sep 17 00:00:00 2001 From: Andrej730 Date: Mon, 8 Dec 2025 13:50:19 +0100 Subject: [PATCH 300/638] Update PCbuild/readme.txt to correct the default platform (GH-142337) --- PCbuild/readme.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 27c0d382281bdb..313982ed28a5dc 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -6,7 +6,7 @@ Quick Start Guide 1a. Optionally install Python 3.10 or later. If not installed, get_externals.bat (via build.bat) will download and use Python via NuGet. -2. Run "build.bat" to build Python in 32-bit Release configuration. +2. Run "build.bat" to build Python in 64-bit Release configuration. 3. (Optional, but recommended) Run the test suite with "rt.bat -q". From 0b8c348f2756c193d6bd2618cadbb90b2f218ccc Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 8 Dec 2025 14:00:31 +0100 Subject: [PATCH 301/638] Fix pyflakes warnings: variable is assigned to but never used (#142294) Example of fixed warning: Lib/netrc.py:98:13: local variable 'toplevel' is assigned to but never used --- Lib/_py_warnings.py | 1 - Lib/_threading_local.py | 2 +- Lib/asyncio/base_events.py | 4 ++-- Lib/asyncio/proactor_events.py | 2 +- Lib/asyncio/tools.py | 2 +- Lib/asyncio/unix_events.py | 2 +- Lib/codeop.py | 4 ++-- Lib/compileall.py | 2 +- Lib/concurrent/interpreters/_queues.py | 6 +++--- Lib/dis.py | 1 - Lib/encodings/uu_codec.py | 2 +- Lib/ftplib.py | 8 ++++---- Lib/functools.py | 2 +- Lib/idlelib/pyshell.py | 1 - Lib/idlelib/textview.py | 4 ++-- Lib/imaplib.py | 6 +++--- Lib/inspect.py | 1 - Lib/modulefinder.py | 1 - Lib/multiprocessing/connection.py | 3 +-- Lib/netrc.py | 2 +- Lib/ntpath.py | 2 +- Lib/optparse.py | 2 +- Lib/pickle.py | 5 ++--- Lib/platform.py | 3 +-- Lib/pyclbr.py | 1 - Lib/re/_parser.py | 1 - Lib/subprocess.py | 2 +- Lib/tempfile.py | 2 +- Lib/tokenize.py | 2 +- Lib/turtle.py | 4 ++-- Lib/urllib/parse.py | 2 +- Lib/venv/__init__.py | 1 - 32 files changed, 36 insertions(+), 47 deletions(-) diff --git a/Lib/_py_warnings.py b/Lib/_py_warnings.py index 67c74fdd2d0b42..d5a9cec86f3674 100644 --- a/Lib/_py_warnings.py +++ b/Lib/_py_warnings.py @@ -563,7 +563,6 @@ def warn_explicit(message, category, filename, lineno, else: text = message message = category(message) - modules = None key = (text, category, lineno) with _wm._lock: if registry is None: diff --git a/Lib/_threading_local.py b/Lib/_threading_local.py index 0b9e5d3bbf6ef6..2af3885458b54f 100644 --- a/Lib/_threading_local.py +++ b/Lib/_threading_local.py @@ -57,7 +57,7 @@ def thread_deleted(_, idt=idt): # as soon as the OS-level thread ends instead. local = wrlocal() if local is not None: - dct = local.dicts.pop(idt) + local.dicts.pop(idt) wrlocal = ref(self, local_deleted) wrthread = ref(thread, thread_deleted) thread.__dict__[key] = wrlocal diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 8cbb71f708537f..6619c87bcf5b93 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -949,7 +949,7 @@ async def sock_sendfile(self, sock, file, offset=0, count=None, try: return await self._sock_sendfile_native(sock, file, offset, count) - except exceptions.SendfileNotAvailableError as exc: + except exceptions.SendfileNotAvailableError: if not fallback: raise return await self._sock_sendfile_fallback(sock, file, @@ -1270,7 +1270,7 @@ async def sendfile(self, transport, file, offset=0, count=None, try: return await self._sendfile_native(transport, file, offset, count) - except exceptions.SendfileNotAvailableError as exc: + except exceptions.SendfileNotAvailableError: if not fallback: raise diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py index f404273c3ae5c1..3fa93b14a6787f 100644 --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -733,7 +733,7 @@ async def sock_accept(self, sock): async def _sock_sendfile_native(self, sock, file, offset, count): try: fileno = file.fileno() - except (AttributeError, io.UnsupportedOperation) as err: + except (AttributeError, io.UnsupportedOperation): raise exceptions.SendfileNotAvailableError("not a regular file") try: fsize = os.fstat(fileno).st_size diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index 1d463ea09ba5b8..f9b8a4ee56c5c1 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -244,7 +244,7 @@ def _get_awaited_by_tasks(pid: int) -> list: e = e.__context__ print(f"Error retrieving tasks: {e}") sys.exit(1) - except PermissionError as e: + except PermissionError: exit_with_permission_help_text() diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index 1c1458127db5ac..49e8067ee7b4e5 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -359,7 +359,7 @@ async def _sock_sendfile_native(self, sock, file, offset, count): "os.sendfile() is not available") try: fileno = file.fileno() - except (AttributeError, io.UnsupportedOperation) as err: + except (AttributeError, io.UnsupportedOperation): raise exceptions.SendfileNotAvailableError("not a regular file") try: fsize = os.fstat(fileno).st_size diff --git a/Lib/codeop.py b/Lib/codeop.py index 8cac00442d99e3..40e88423119bc4 100644 --- a/Lib/codeop.py +++ b/Lib/codeop.py @@ -66,9 +66,9 @@ def _maybe_compile(compiler, source, filename, symbol, flags): try: compiler(source + "\n", filename, symbol, flags=flags) return None - except _IncompleteInputError as e: + except _IncompleteInputError: return None - except SyntaxError as e: + except SyntaxError: pass # fallthrough diff --git a/Lib/compileall.py b/Lib/compileall.py index 67fe370451e1ef..9519a5ac16f024 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -223,7 +223,7 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, cfile = importlib.util.cache_from_source(fullname) opt_cfiles[opt_level] = cfile - head, tail = name[:-3], name[-3:] + tail = name[-3:] if tail == '.py': if not force: try: diff --git a/Lib/concurrent/interpreters/_queues.py b/Lib/concurrent/interpreters/_queues.py index b5cc0b8944940d..ee159d7de63827 100644 --- a/Lib/concurrent/interpreters/_queues.py +++ b/Lib/concurrent/interpreters/_queues.py @@ -223,7 +223,7 @@ def put(self, obj, block=True, timeout=None, *, while True: try: _queues.put(self._id, obj, unboundop) - except QueueFull as exc: + except QueueFull: if timeout is not None and time.time() >= end: raise # re-raise time.sleep(_delay) @@ -258,7 +258,7 @@ def get(self, block=True, timeout=None, *, while True: try: obj, unboundop = _queues.get(self._id) - except QueueEmpty as exc: + except QueueEmpty: if timeout is not None and time.time() >= end: raise # re-raise time.sleep(_delay) @@ -277,7 +277,7 @@ def get_nowait(self): """ try: obj, unboundop = _queues.get(self._id) - except QueueEmpty as exc: + except QueueEmpty: raise # re-raise if unboundop is not None: assert obj is None, repr(obj) diff --git a/Lib/dis.py b/Lib/dis.py index d6d2c1386dd785..8c257d118fb23b 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -530,7 +530,6 @@ def print_instruction_line(self, instr, mark_as_current): fields.append(instr.opname.ljust(_OPNAME_WIDTH)) # Column: Opcode argument if instr.arg is not None: - arg = repr(instr.arg) # If opname is longer than _OPNAME_WIDTH, we allow it to overflow into # the space reserved for oparg. This results in fewer misaligned opargs # in the disassembly output. diff --git a/Lib/encodings/uu_codec.py b/Lib/encodings/uu_codec.py index 4e58c62fe9ef0f..4f8704016e2131 100644 --- a/Lib/encodings/uu_codec.py +++ b/Lib/encodings/uu_codec.py @@ -56,7 +56,7 @@ def uu_decode(input, errors='strict'): break try: data = binascii.a2b_uu(s) - except binascii.Error as v: + except binascii.Error: # Workaround for broken uuencoders by /Fredrik Lundh nbytes = (((s[0]-32) & 63) * 4 + 5) // 3 data = binascii.a2b_uu(s[:nbytes]) diff --git a/Lib/ftplib.py b/Lib/ftplib.py index 50771e8c17c250..640acc64f620cc 100644 --- a/Lib/ftplib.py +++ b/Lib/ftplib.py @@ -314,9 +314,9 @@ def makeport(self): port = sock.getsockname()[1] # Get proper port host = self.sock.getsockname()[0] # Get proper host if self.af == socket.AF_INET: - resp = self.sendport(host, port) + self.sendport(host, port) else: - resp = self.sendeprt(host, port) + self.sendeprt(host, port) if self.timeout is not _GLOBAL_DEFAULT_TIMEOUT: sock.settimeout(self.timeout) return sock @@ -455,7 +455,7 @@ def retrlines(self, cmd, callback = None): """ if callback is None: callback = print_line - resp = self.sendcmd('TYPE A') + self.sendcmd('TYPE A') with self.transfercmd(cmd) as conn, \ conn.makefile('r', encoding=self.encoding) as fp: while 1: @@ -951,7 +951,7 @@ def test(): elif file[:2] == '-d': cmd = 'CWD' if file[2:]: cmd = cmd + ' ' + file[2:] - resp = ftp.sendcmd(cmd) + ftp.sendcmd(cmd) elif file == '-p': ftp.set_pasv(not ftp.passiveserver) else: diff --git a/Lib/functools.py b/Lib/functools.py index 8063eb5ffc3304..836eb680ccd4d4 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -687,7 +687,7 @@ def wrapper(*args, **kwds): # still adjusting the links. root = oldroot[NEXT] oldkey = root[KEY] - oldresult = root[RESULT] + oldresult = root[RESULT] # noqa: F841 root[KEY] = root[RESULT] = None # Now update the cache dictionary. diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index 1b7c2af1a923d7..b80c8e56c92810 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -498,7 +498,6 @@ def restart_subprocess(self, with_cwd=False, filename=''): self.rpcclt.close() self.terminate_subprocess() console = self.tkconsole - was_executing = console.executing console.executing = False self.spawn_subprocess() try: diff --git a/Lib/idlelib/textview.py b/Lib/idlelib/textview.py index 23f0f4cb5027ec..0f719a06883ad7 100644 --- a/Lib/idlelib/textview.py +++ b/Lib/idlelib/textview.py @@ -129,8 +129,8 @@ def __init__(self, parent, title, contents, modal=True, wrap=WORD, self.title(title) self.viewframe = ViewFrame(self, contents, wrap=wrap) self.protocol("WM_DELETE_WINDOW", self.ok) - self.button_ok = button_ok = Button(self, text='Close', - command=self.ok, takefocus=False) + self.button_ok = Button(self, text='Close', + command=self.ok, takefocus=False) self.viewframe.pack(side='top', expand=True, fill='both') self.is_modal = modal diff --git a/Lib/imaplib.py b/Lib/imaplib.py index c176736548188c..22a0afcd981519 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -808,7 +808,7 @@ def proxyauth(self, user): """ name = 'PROXYAUTH' - return self._simple_command('PROXYAUTH', user) + return self._simple_command(name, user) def rename(self, oldmailbox, newmailbox): @@ -1310,7 +1310,7 @@ def _get_tagged_response(self, tag, expect_bye=False): try: self._get_response() - except self.abort as val: + except self.abort: if __debug__: if self.debug >= 1: self.print_log() @@ -1867,7 +1867,7 @@ def Time2Internaldate(date_time): try: optlist, args = getopt.getopt(sys.argv[1:], 'd:s:') - except getopt.error as val: + except getopt.error: optlist, args = (), () stream_command = None diff --git a/Lib/inspect.py b/Lib/inspect.py index 8e7511b3af015f..ff462750888c88 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2140,7 +2140,6 @@ def _signature_strip_non_python_syntax(signature): current_parameter = 0 OP = token.OP - ERRORTOKEN = token.ERRORTOKEN # token stream always starts with ENCODING token, skip it t = next(token_stream) diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py index b115d99ab30ff1..7fb19a5c5d1805 100644 --- a/Lib/modulefinder.py +++ b/Lib/modulefinder.py @@ -400,7 +400,6 @@ def scan_opcodes(self, co): yield "relative_import", (level, fromlist, name) def scan_code(self, co, m): - code = co.co_code scanner = self.scan_opcodes for what, args in scanner(co): if what == "store": diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index fc00d2861260a8..64ec53884aeb5d 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -709,8 +709,7 @@ def accept(self): # written data and then disconnected -- see Issue 14725. else: try: - res = _winapi.WaitForMultipleObjects( - [ov.event], False, INFINITE) + _winapi.WaitForMultipleObjects([ov.event], False, INFINITE) except: ov.cancel() _winapi.CloseHandle(handle) diff --git a/Lib/netrc.py b/Lib/netrc.py index 2f502c1d53364f..750b5071e3c65f 100644 --- a/Lib/netrc.py +++ b/Lib/netrc.py @@ -95,7 +95,7 @@ def _parse(self, file, fp, default_netrc): while 1: # Look for a machine, default, or macdef top-level keyword saved_lineno = lexer.lineno - toplevel = tt = lexer.get_token() + tt = lexer.get_token() if not tt: break elif tt[0] == '#': diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 77d2bf86a5f09d..7d637325240f1c 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -726,7 +726,7 @@ def realpath(path, /, *, strict=False): try: if _getfinalpathname(spath) == path: path = spath - except ValueError as ex: + except ValueError: # Unexpected, as an invalid path should not have gained a prefix # at any point, but we ignore this error just in case. pass diff --git a/Lib/optparse.py b/Lib/optparse.py index 02ff7140882ed6..5ff7f74754f9c1 100644 --- a/Lib/optparse.py +++ b/Lib/optparse.py @@ -1372,7 +1372,7 @@ def parse_args(self, args=None, values=None): self.values = values try: - stop = self._process_args(largs, rargs, values) + self._process_args(largs, rargs, values) except (BadOptionError, OptionValueError) as err: self.error(str(err)) diff --git a/Lib/pickle.py b/Lib/pickle.py index f3025776623d2c..71c12c50f7f035 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -1169,7 +1169,6 @@ def save_frozenset(self, obj): def save_global(self, obj, name=None): write = self.write - memo = self.memo if name is None: name = getattr(obj, '__qualname__', None) @@ -1756,7 +1755,7 @@ def load_binget(self): i = self.read(1)[0] try: self.append(self.memo[i]) - except KeyError as exc: + except KeyError: msg = f'Memo value not found at index {i}' raise UnpicklingError(msg) from None dispatch[BINGET[0]] = load_binget @@ -1765,7 +1764,7 @@ def load_long_binget(self): i, = unpack(' Date: Mon, 8 Dec 2025 07:05:13 -0600 Subject: [PATCH 302/638] gh-140125: Increase object recursion depth for `test_json` from 200k to 500k (#142226) Co-authored-by: Victor Stinner --- Lib/test/test_json/test_recursion.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_json/test_recursion.py b/Lib/test/test_json/test_recursion.py index 5d7b56ff9ad285..40a0baa53f0c3b 100644 --- a/Lib/test/test_json/test_recursion.py +++ b/Lib/test/test_json/test_recursion.py @@ -71,7 +71,7 @@ def default(self, o): @support.skip_emscripten_stack_overflow() @support.skip_wasi_stack_overflow() def test_highly_nested_objects_decoding(self): - very_deep = 200000 + very_deep = 500_000 # test that loading highly-nested objects doesn't segfault when C # accelerations are used. See #12017 with self.assertRaises(RecursionError): @@ -90,7 +90,7 @@ def test_highly_nested_objects_decoding(self): def test_highly_nested_objects_encoding(self): # See #12051 l, d = [], {} - for x in range(200_000): + for x in range(500_000): l, d = [l], {'k':d} with self.assertRaises(RecursionError): with support.infinite_recursion(5000): From c4ccaf4b1051b3c1ae0138a9c92657606f578fbd Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Mon, 8 Dec 2025 23:47:19 +0900 Subject: [PATCH 303/638] gh-141770: Annotate anonymous mmap usage if "-X dev" is used (gh-142079) --- Doc/whatsnew/3.15.rst | 6 +++ Include/internal/pycore_mmap.h | 45 +++++++++++++++++++ Makefile.pre.in | 1 + ...-11-29-18-14-28.gh-issue-141770.JURnvg.rst | 2 + Modules/_ctypes/malloc_closure.c | 5 ++- Modules/mmapmodule.c | 2 + Objects/obmalloc.c | 2 + PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 ++ Python/jit.c | 4 ++ Python/perf_jit_trampoline.c | 2 + Python/perf_trampoline.c | 2 + configure | 19 ++++++++ configure.ac | 7 +++ pyconfig.h.in | 7 +++ 15 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 Include/internal/pycore_mmap.h create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-11-29-18-14-28.gh-issue-141770.JURnvg.rst diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 1bd82545e588fa..0c892e63393ad1 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -1236,6 +1236,12 @@ Build changes modules that are missing or packaged separately. (Contributed by Stan Ulbrych and Petr Viktorin in :gh:`139707`.) +* Annotating anonymous mmap usage is now supported if Linux kernel supports + :manpage:`PR_SET_VMA_ANON_NAME ` (Linux 5.17 or newer). + Annotations are visible in ``/proc//maps`` if the kernel supports the feature + and :option:`-X dev <-X>` is passed to the Python or Python is built in :ref:`debug mode `. + (Contributed by Donghee Na in :gh:`141770`) + Porting to Python 3.15 ====================== diff --git a/Include/internal/pycore_mmap.h b/Include/internal/pycore_mmap.h new file mode 100644 index 00000000000000..214fd4362a55fe --- /dev/null +++ b/Include/internal/pycore_mmap.h @@ -0,0 +1,45 @@ +#ifndef Py_INTERNAL_MMAP_H +#define Py_INTERNAL_MMAP_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include "pycore_pystate.h" + +#if defined(HAVE_PR_SET_VMA_ANON_NAME) && defined(__linux__) +# include +# include +#endif + +#if defined(HAVE_PR_SET_VMA_ANON_NAME) && defined(__linux__) +static inline void +_PyAnnotateMemoryMap(void *addr, size_t size, const char *name) +{ +#ifndef Py_DEBUG + if (!_Py_GetConfig()->dev_mode) { + return; + } +#endif + assert(strlen(name) < 80); + int old_errno = errno; + prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, (unsigned long)addr, size, name); + /* Ignore errno from prctl */ + /* See: https://bugzilla.redhat.com/show_bug.cgi?id=2302746 */ + errno = old_errno; +} +#else +static inline void +_PyAnnotateMemoryMap(void *Py_UNUSED(addr), size_t Py_UNUSED(size), const char *Py_UNUSED(name)) +{ +} +#endif + +#ifdef __cplusplus +} +#endif +#endif // !Py_INTERNAL_MMAP_H diff --git a/Makefile.pre.in b/Makefile.pre.in index f3086ec1462b6b..2554114fff6d6c 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1378,6 +1378,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_long.h \ $(srcdir)/Include/internal/pycore_memoryobject.h \ $(srcdir)/Include/internal/pycore_mimalloc.h \ + $(srcdir)/Include/internal/pycore_mmap.h \ $(srcdir)/Include/internal/pycore_modsupport.h \ $(srcdir)/Include/internal/pycore_moduleobject.h \ $(srcdir)/Include/internal/pycore_namespace.h \ diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-11-29-18-14-28.gh-issue-141770.JURnvg.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-29-18-14-28.gh-issue-141770.JURnvg.rst new file mode 100644 index 00000000000000..3a5c0fd70edb1e --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-11-29-18-14-28.gh-issue-141770.JURnvg.rst @@ -0,0 +1,2 @@ +Annotate anonymous mmap usage only when supported by the +Linux kernel and if ``-X dev`` is used or Python is built in debug mode. Patch by Donghee Na. diff --git a/Modules/_ctypes/malloc_closure.c b/Modules/_ctypes/malloc_closure.c index db405acf8727b5..62c7aa5d6affbf 100644 --- a/Modules/_ctypes/malloc_closure.c +++ b/Modules/_ctypes/malloc_closure.c @@ -14,6 +14,7 @@ # endif #endif #include "ctypes.h" +#include "pycore_mmap.h" // _PyAnnotateMemoryMap() /* BLOCKSIZE can be adjusted. Larger blocksize will take a larger memory overhead, but allocate less blocks from the system. It may be that some @@ -74,14 +75,16 @@ static void more_core(void) if (item == NULL) return; #else + size_t mem_size = count * sizeof(ITEM); item = (ITEM *)mmap(NULL, - count * sizeof(ITEM), + mem_size, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (item == (void *)MAP_FAILED) return; + _PyAnnotateMemoryMap(item, mem_size, "cpython:ctypes"); #endif #ifdef MALLOC_CLOSURE_DEBUG diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index ac8521f8aa9b6e..37003020de2688 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -26,6 +26,7 @@ #include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() #include "pycore_bytesobject.h" // _PyBytes_Find() #include "pycore_fileutils.h" // _Py_stat_struct +#include "pycore_mmap.h" // _PyAnnotateMemoryMap() #include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include // offsetof() @@ -1951,6 +1952,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) PyErr_SetFromErrno(PyExc_OSError); return NULL; } + _PyAnnotateMemoryMap(m_obj->data, map_size, "cpython:mmap"); m_obj->access = (access_mode)access; return (PyObject *)m_obj; } diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 2b95ebbf8e5ac0..b1f9fa2e692265 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -2,6 +2,7 @@ #include "Python.h" #include "pycore_interp.h" // _PyInterpreterState_HasFeature +#include "pycore_mmap.h" // _PyAnnotateMemoryMap() #include "pycore_object.h" // _PyDebugAllocatorStats() definition #include "pycore_obmalloc.h" #include "pycore_obmalloc_init.h" @@ -467,6 +468,7 @@ _PyMem_ArenaAlloc(void *Py_UNUSED(ctx), size_t size) if (ptr == MAP_FAILED) return NULL; assert(ptr != NULL); + _PyAnnotateMemoryMap(ptr, size, "cpython:pymalloc"); return ptr; #else return malloc(size); diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 85363949c2344f..dcfb75ce162b2f 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -277,6 +277,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 17999690990fb9..247f4b5a784f9c 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -735,6 +735,9 @@ Include\internal + + Include\internal + Include\internal diff --git a/Python/jit.c b/Python/jit.c index 47d3d7a5d27180..7106db8a99a77a 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -16,6 +16,7 @@ #include "pycore_intrinsics.h" #include "pycore_list.h" #include "pycore_long.h" +#include "pycore_mmap.h" #include "pycore_opcode_metadata.h" #include "pycore_opcode_utils.h" #include "pycore_optimizer.h" @@ -75,6 +76,9 @@ jit_alloc(size_t size) int prot = PROT_READ | PROT_WRITE; unsigned char *memory = mmap(NULL, size, prot, flags, -1, 0); int failed = memory == MAP_FAILED; + if (!failed) { + _PyAnnotateMemoryMap(memory, size, "cpython:jit"); + } #endif if (failed) { jit_error("unable to allocate memory"); diff --git a/Python/perf_jit_trampoline.c b/Python/perf_jit_trampoline.c index 8732be973616d4..af7d8f9f1ec0ae 100644 --- a/Python/perf_jit_trampoline.c +++ b/Python/perf_jit_trampoline.c @@ -61,6 +61,7 @@ #include "pycore_ceval.h" // _PyPerf_Callbacks #include "pycore_frame.h" #include "pycore_interp.h" +#include "pycore_mmap.h" // _PyAnnotateMemoryMap() #include "pycore_runtime.h" // _PyRuntime #ifdef PY_HAVE_PERF_TRAMPOLINE @@ -1085,6 +1086,7 @@ static void* perf_map_jit_init(void) { close(fd); return NULL; // Memory mapping failed } + _PyAnnotateMemoryMap(perf_jit_map_state.mapped_buffer, page_size, "cpython:perf_jit_trampoline"); #endif perf_jit_map_state.mapped_size = page_size; diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index 987e8d2a11a659..669a47ae17377a 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -132,6 +132,7 @@ any DWARF information available for them). #include "Python.h" #include "pycore_ceval.h" // _PyPerf_Callbacks #include "pycore_interpframe.h" // _PyFrame_GetCode() +#include "pycore_mmap.h" // _PyAnnotateMemoryMap() #include "pycore_runtime.h" // _PyRuntime @@ -290,6 +291,7 @@ new_code_arena(void) perf_status = PERF_STATUS_FAILED; return -1; } + _PyAnnotateMemoryMap(memory, mem_size, "cpython:perf_trampoline"); void *start = &_Py_trampoline_func_start; void *end = &_Py_trampoline_func_end; size_t code_size = end - start; diff --git a/configure b/configure index 7561fb9c7ad90e..4f9b9b21ca395e 100755 --- a/configure +++ b/configure @@ -23947,6 +23947,25 @@ printf "%s\n" "#define HAVE_UT_NAMESIZE 1" >>confdefs.h fi +ac_fn_check_decl "$LINENO" "PR_SET_VMA_ANON_NAME" "ac_cv_have_decl_PR_SET_VMA_ANON_NAME" "#include + #include +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_PR_SET_VMA_ANON_NAME" = xyes +then : + ac_have_decl=1 +else case e in #( + e) ac_have_decl=0 ;; +esac +fi +printf "%s\n" "#define HAVE_DECL_PR_SET_VMA_ANON_NAME $ac_have_decl" >>confdefs.h +if test $ac_have_decl = 1 +then : + +printf "%s\n" "#define HAVE_PR_SET_VMA_ANON_NAME 1" >>confdefs.h + +fi + + # check for openpty, login_tty, and forkpty diff --git a/configure.ac b/configure.ac index fa24bc78a2645a..046c046ffb5d10 100644 --- a/configure.ac +++ b/configure.ac @@ -5583,6 +5583,13 @@ AC_CHECK_DECLS([UT_NAMESIZE], [], [@%:@include ]) +AC_CHECK_DECLS([PR_SET_VMA_ANON_NAME], + [AC_DEFINE([HAVE_PR_SET_VMA_ANON_NAME], [1], + [Define if you have the 'PR_SET_VMA_ANON_NAME' constant.])], + [], + [@%:@include + @%:@include ]) + # check for openpty, login_tty, and forkpty AC_CHECK_FUNCS([openpty], [], diff --git a/pyconfig.h.in b/pyconfig.h.in index 8a9f5ca8ec826d..aabf9f0be8da55 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -228,6 +228,10 @@ /* Define to 1 if you have the header file. */ #undef HAVE_DB_H +/* Define to 1 if you have the declaration of 'PR_SET_VMA_ANON_NAME', and to 0 + if you don't. */ +#undef HAVE_DECL_PR_SET_VMA_ANON_NAME + /* Define to 1 if you have the declaration of 'RTLD_DEEPBIND', and to 0 if you don't. */ #undef HAVE_DECL_RTLD_DEEPBIND @@ -996,6 +1000,9 @@ /* Define if your compiler supports function prototype */ #undef HAVE_PROTOTYPES +/* Define if you have the 'PR_SET_VMA_ANON_NAME' constant. */ +#undef HAVE_PR_SET_VMA_ANON_NAME + /* Define to 1 if you have the 'pthread_condattr_setclock' function. */ #undef HAVE_PTHREAD_CONDATTR_SETCLOCK From 9d39c02498208ea9253492c5940d57bdb7627094 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Mon, 8 Dec 2025 08:54:47 -0800 Subject: [PATCH 304/638] Temporarily allow CI failures for iOS (#142365) iOS tests are flaky right now. Based on the convo in Discord, it seems like allowing failures is the best option. Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3d889fa128e261..c43ec75fba3619 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -706,6 +706,7 @@ jobs: uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe with: allowed-failures: >- + build-ios, build-windows-msi, build-ubuntu-ssltests-awslc, build-ubuntu-ssltests-openssl, From f2fba4c99aadde52858212b1d552d9f2425bb568 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Mon, 8 Dec 2025 12:14:50 -0500 Subject: [PATCH 305/638] gh-124379: Document _PyStackRef (gh-142321) --- Include/internal/pycore_stackref.h | 7 --- InternalDocs/README.md | 2 + InternalDocs/stackrefs.md | 80 ++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 7 deletions(-) create mode 100644 InternalDocs/stackrefs.md diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index e59611c07fa793..c86beebe6554c3 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -479,13 +479,6 @@ PyStackRef_AsPyObjectBorrow(_PyStackRef stackref) #define PyStackRef_IsDeferred(ref) (((ref).bits & Py_TAG_BITS) == Py_TAG_DEFERRED) -static inline PyObject * -PyStackRef_NotDeferred_AsPyObject(_PyStackRef stackref) -{ - assert(!PyStackRef_IsDeferred(stackref)); - return (PyObject *)stackref.bits; -} - static inline PyObject * PyStackRef_AsPyObjectSteal(_PyStackRef stackref) { diff --git a/InternalDocs/README.md b/InternalDocs/README.md index a6e2df5ae4a9c3..06f67b3cfc124a 100644 --- a/InternalDocs/README.md +++ b/InternalDocs/README.md @@ -36,6 +36,8 @@ Program Execution - [The Bytecode Interpreter](interpreter.md) +- [Stack references (_PyStackRef)](stackrefs.md) + - [The JIT](jit.md) - [Garbage Collector Design](garbage_collector.md) diff --git a/InternalDocs/stackrefs.md b/InternalDocs/stackrefs.md new file mode 100644 index 00000000000000..2d8810262d45f7 --- /dev/null +++ b/InternalDocs/stackrefs.md @@ -0,0 +1,80 @@ +# Stack references (`_PyStackRef`) + +Stack references are the interpreter's tagged representation of values on the evaluation stack. +They carry metadata to track ownership and support optimizations such as tagged small ints. + +## Shape and tagging + +- A `_PyStackRef` is a tagged pointer-sized value (see `Include/internal/pycore_stackref.h`). +- Tag bits distinguish three cases: + - `Py_TAG_REFCNT` unset - reference count lives on the pointed-to object. + - `Py_TAG_REFCNT` set - ownership is "borrowed" (no refcount to drop on close) or the object is immortal. + - `Py_INT_TAG` set - tagged small integer stored directly in the stackref (no heap allocation). +- Special constants: `PyStackRef_NULL`, `PyStackRef_ERROR`, and embedded `None`/`True`/`False`. + +In GIL builds, most objects carry their refcount; tagged borrowed refs skip decref on close. In free +threading builds, the tag is also used to mark deferred refcounted objects so the GC can see them and +to avoid refcount contention on commonly shared objects. + +## Converting to and from PyObject* + +Three conversions control ownership: + +- `PyStackRef_FromPyObjectNew(obj)` - create a new reference (INCREF if mortal). +- `PyStackRef_FromPyObjectSteal(obj)` - take over ownership without changing the count unless the + object is immortal. +- `PyStackRef_FromPyObjectBorrow(obj)` - create a borrowed stackref (never decref on close). + +The `obj` argument must not be `NULL`. + +Going back to `PyObject*` mirrors this: + +- `PyStackRef_AsPyObjectBorrow(ref)` - borrow the underlying pointer +- `PyStackRef_AsPyObjectSteal(ref)` - transfer ownership from the stackref; if ref is borrowed or + deferred, this creates a new owning `PyObject*` reference. +- `PyStackRef_AsPyObjectNew(ref)` - create a new owning reference + +Only `PyStackRef_AsPyObjectBorrow` allows ref to be `PyStackRef_NULL`. + +## Operations on stackrefs + +The interpreter treats `_PyStackRef` as the unit of stack storage. Ownership must be managed with +the stackref primitives: + +- `PyStackRef_DUP` - like `Py_NewRef` for stackrefs; preserves the original. +- `PyStackRef_Borrow` - create a borrowed stackref from another stackref. +- `PyStackRef_CLOSE` / `PyStackRef_XCLOSE` - like `Py_DECREF`; invalidates the stackref. +- `PyStackRef_CLEAR` - like `Py_CLEAR`; closes and sets the stackref to `PyStackRef_NULL` +- `PyStackRef_MakeHeapSafe` - converts borrowed reference to owning reference + +Borrow tracking (for debug builds with `Py_STACKREF_DEBUG`) records who you borrowed from and reports +double-close, leaked borrows, or use-after-close via fatal errors. + +## Borrow-friendly opcodes + +The interpreter can push borrowed references directly. For example, `LOAD_FAST_BORROW` loads a local +variable as a borrowed `_PyStackRef`, avoiding both INCREF and DECREF for the temporary lifetime on +the evaluation stack. + +## Tagged integers on the stack + +Small ints can be stored inline with `Py_INT_TAG`, so no heap object is involved. Helpers like +`PyStackRef_TagInt`, `PyStackRef_UntagInt`, and `PyStackRef_IncrementTaggedIntNoOverflow` operate on +these values. Type checks use `PyStackRef_IsTaggedInt` and `PyStackRef_LongCheck`. + +## Free threading considerations + +With `Py_GIL_DISABLED`, `Py_TAG_DEFERRED` is an alias for `Py_TAG_REFCNT`. +Objects that support deferred reference counting can be pushed to the evaluation +stack and stored in local variables without directly incrementing the reference +count because they are only freed during cyclic garbage collection. This avoids +reference count contention on commonly shared objects such as methods and types. The GC +scans each thread's locals and evaluation stack to keep objects that use +deferred reference counting alive. + +## Debugging support + +`Py_STACKREF_DEBUG` builds replace the inline tags with table-backed IDs so the runtime can track +creation sites, borrows, closes, and leaks. Enabling `Py_STACKREF_CLOSE_DEBUG` additionally records +double closes. The tables live on `PyInterpreterState` and are initialized in `pystate.c`; helper +routines reside in `Python/stackrefs.c`. From 37988c57ea244b0ed2f969e9c6039710dfe8f31d Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Mon, 8 Dec 2025 12:22:13 -0500 Subject: [PATCH 306/638] gh-123241: Document restrictions for `tp_traverse` implementations (gh-142272) --- Doc/c-api/gcsupport.rst | 4 ++++ Doc/c-api/typeobj.rst | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/Doc/c-api/gcsupport.rst b/Doc/c-api/gcsupport.rst index f6fa52b36c5ab3..fed795b1e8c963 100644 --- a/Doc/c-api/gcsupport.rst +++ b/Doc/c-api/gcsupport.rst @@ -232,6 +232,10 @@ The :c:member:`~PyTypeObject.tp_traverse` handler must have the following type: object argument. If *visit* returns a non-zero value that value should be returned immediately. + The traversal function must not have any side effects. Implementations + may not modify the reference counts of any Python objects nor create or + destroy any Python objects. + To simplify writing :c:member:`~PyTypeObject.tp_traverse` handlers, a :c:func:`Py_VISIT` macro is provided. In order to use this macro, the :c:member:`~PyTypeObject.tp_traverse` implementation must name its arguments exactly *visit* and *arg*: diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 49fe02d919df8b..efac86078f9af5 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -1569,6 +1569,11 @@ and :c:data:`PyType_Type` effectively act as defaults.) but the instance has no strong reference to the elements inside it, as they are allowed to be removed even if the instance is still alive). + .. warning:: + The traversal function must not have any side effects. It must not + modify the reference counts of any Python objects nor create or destroy + any Python objects. + Note that :c:func:`Py_VISIT` requires the *visit* and *arg* parameters to :c:func:`!local_traverse` to have these specific names; don't name them just anything. From e0451ceef8c18efbc378b959b59c4681d92cd686 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 8 Dec 2025 17:57:11 +0000 Subject: [PATCH 307/638] GH-139757: JIT: Remove redundant branches to jumps in the assembly optimizer (GH-140800) JIT: Remove redundant branches to jump in the assembly optimizer * Refactor JIT assembly optimizer making instructions instances not just strings * Remove redundant jumps and branches where legal to do so * Modifies _BINARY_OP_SUBSCR_STR_INT to avoid excessive inlining depth --- Python/bytecodes.c | 7 +- Python/executor_cases.c.h | 5 +- Python/generated_cases.c.h | 5 +- Python/jit.c | 3 +- Tools/cases_generator/analyzer.py | 2 + Tools/jit/_optimizers.py | 260 ++++++++++++++++++++++-------- Tools/jit/_stencils.py | 2 +- Tools/jit/_targets.py | 10 +- 8 files changed, 220 insertions(+), 74 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 4ba255d28bdcf6..6411049796bf12 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -982,9 +982,10 @@ dummy_func( DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index); - // Specialize for reading an ASCII character from any string: - Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c); + // Specialize for reading an ASCII character from an ASCII string: + DEOPT_IF(!PyUnicode_IS_COMPACT_ASCII(str)); + uint8_t c = PyUnicode_1BYTE_DATA(str)[index]; + assert(c < 128); STAT_INC(BINARY_OP, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 7273a87681b4dd..079d31da6c1b7a 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1502,11 +1502,12 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) { + if (!PyUnicode_IS_COMPACT_ASCII(str)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } + uint8_t c = PyUnicode_1BYTE_DATA(str)[index]; + assert(c < 128); STAT_INC(BINARY_OP, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 68d73cccec4d6b..3d5bf75ac0acae 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -892,12 +892,13 @@ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } - Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) { + if (!PyUnicode_IS_COMPACT_ASCII(str)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } + uint8_t c = PyUnicode_1BYTE_DATA(str)[index]; + assert(c < 128); STAT_INC(BINARY_OP, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); diff --git a/Python/jit.c b/Python/jit.c index 7106db8a99a77a..b0d53d156fa440 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -185,6 +185,7 @@ set_bits(uint32_t *loc, uint8_t loc_start, uint64_t value, uint8_t value_start, #define IS_AARCH64_ADRP(I) (((I) & 0x9F000000) == 0x90000000) #define IS_AARCH64_BRANCH(I) (((I) & 0x7C000000) == 0x14000000) #define IS_AARCH64_BRANCH_COND(I) (((I) & 0x7C000000) == 0x54000000) +#define IS_AARCH64_BRANCH_ZERO(I) (((I) & 0x7E000000) == 0x34000000) #define IS_AARCH64_TEST_AND_BRANCH(I) (((I) & 0x7E000000) == 0x36000000) #define IS_AARCH64_LDR_OR_STR(I) (((I) & 0x3B000000) == 0x39000000) #define IS_AARCH64_MOV(I) (((I) & 0x9F800000) == 0x92800000) @@ -352,7 +353,7 @@ void patch_aarch64_19r(unsigned char *location, uint64_t value) { uint32_t *loc32 = (uint32_t *)location; - assert(IS_AARCH64_BRANCH_COND(*loc32)); + assert(IS_AARCH64_BRANCH_COND(*loc32) || IS_AARCH64_BRANCH_ZERO(*loc32)); value -= (uintptr_t)location; // Check that we're not out of range of 21 signed bits: assert((int64_t)value >= -(1 << 20)); diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 93aa4899fe6ec8..9293a649e8a0ec 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -614,6 +614,8 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "PyUnicode_Concat", "PyUnicode_GET_LENGTH", "PyUnicode_READ_CHAR", + "PyUnicode_IS_COMPACT_ASCII", + "PyUnicode_1BYTE_DATA", "Py_ARRAY_LENGTH", "Py_FatalError", "Py_INCREF", diff --git a/Tools/jit/_optimizers.py b/Tools/jit/_optimizers.py index 0adc550ba5e84c..297b9517f6a27a 100644 --- a/Tools/jit/_optimizers.py +++ b/Tools/jit/_optimizers.py @@ -1,6 +1,7 @@ """Low-level optimization of textual assembly.""" import dataclasses +import enum import pathlib import re import typing @@ -65,23 +66,72 @@ # MyPy doesn't understand that a invariant variable can be initialized by a covariant value CUSTOM_AARCH64_BRANCH19: str | None = "CUSTOM_AARCH64_BRANCH19" -# Branches are either b.{cond} or bc.{cond} -_AARCH64_BRANCHES: dict[str, tuple[str | None, str | None]] = { - "b." + cond: (("b." + inverse if inverse else None), CUSTOM_AARCH64_BRANCH19) - for (cond, inverse) in _AARCH64_COND_CODES.items() -} | { - "bc." + cond: (("bc." + inverse if inverse else None), CUSTOM_AARCH64_BRANCH19) - for (cond, inverse) in _AARCH64_COND_CODES.items() +_AARCH64_SHORT_BRANCHES = { + "tbz": "tbnz", + "tbnz": "tbz", } +# Branches are either b.{cond}, bc.{cond}, cbz, cbnz, tbz or tbnz +_AARCH64_BRANCHES: dict[str, tuple[str | None, str | None]] = ( + { + "b." + cond: (("b." + inverse if inverse else None), CUSTOM_AARCH64_BRANCH19) + for (cond, inverse) in _AARCH64_COND_CODES.items() + } + | { + "bc." + cond: (("bc." + inverse if inverse else None), CUSTOM_AARCH64_BRANCH19) + for (cond, inverse) in _AARCH64_COND_CODES.items() + } + | { + "cbz": ("cbnz", CUSTOM_AARCH64_BRANCH19), + "cbnz": ("cbz", CUSTOM_AARCH64_BRANCH19), + } + | {cond: (inverse, None) for (cond, inverse) in _AARCH64_SHORT_BRANCHES.items()} +) + + +@enum.unique +class InstructionKind(enum.Enum): + + JUMP = enum.auto() + LONG_BRANCH = enum.auto() + SHORT_BRANCH = enum.auto() + RETURN = enum.auto() + OTHER = enum.auto() + @dataclasses.dataclass +class Instruction: + kind: InstructionKind + name: str + text: str + target: str | None + + def is_branch(self) -> bool: + return self.kind in (InstructionKind.LONG_BRANCH, InstructionKind.SHORT_BRANCH) + + def update_target(self, target: str) -> "Instruction": + assert self.target is not None + return Instruction( + self.kind, self.name, self.text.replace(self.target, target), target + ) + + def update_name_and_target(self, name: str, target: str) -> "Instruction": + assert self.target is not None + return Instruction( + self.kind, + name, + self.text.replace(self.name, name).replace(self.target, target), + target, + ) + + +@dataclasses.dataclass(eq=False) class _Block: label: str | None = None # Non-instruction lines like labels, directives, and comments: noninstructions: list[str] = dataclasses.field(default_factory=list) # Instruction lines: - instructions: list[str] = dataclasses.field(default_factory=list) + instructions: list[Instruction] = dataclasses.field(default_factory=list) # If this block ends in a jump, where to? target: typing.Self | None = None # The next block in the linked list: @@ -108,6 +158,7 @@ class Optimizer: # Prefixes used to mangle local labels and symbols: label_prefix: str symbol_prefix: str + re_global: re.Pattern[str] # The first block in the linked list: _root: _Block = dataclasses.field(init=False, default_factory=_Block) _labels: dict[str, _Block] = dataclasses.field(init=False, default_factory=dict) @@ -122,27 +173,36 @@ class Optimizer: # Override everything that follows in subclasses: _supports_external_relocations = True _branches: typing.ClassVar[dict[str, tuple[str | None, str | None]]] = {} + # Short branches are instructions that can branch within a micro-op, + # but might not have the reach to branch anywhere within a trace. + _short_branches: typing.ClassVar[dict[str, str]] = {} # Two groups (instruction and target): _re_branch: typing.ClassVar[re.Pattern[str]] = _RE_NEVER_MATCH # One group (target): _re_jump: typing.ClassVar[re.Pattern[str]] = _RE_NEVER_MATCH # No groups: _re_return: typing.ClassVar[re.Pattern[str]] = _RE_NEVER_MATCH + text: str = "" + globals: set[str] = dataclasses.field(default_factory=set) def __post_init__(self) -> None: # Split the code into a linked list of basic blocks. A basic block is an # optional label, followed by zero or more non-instruction lines, # followed by zero or more instruction lines (only the last of which may # be a branch, jump, or return): - text = self._preprocess(self.path.read_text()) + self.text = self._preprocess(self.path.read_text()) block = self._root - for line in text.splitlines(): + for line in self.text.splitlines(): # See if we need to start a new block: if match := self._re_label.match(line): # Label. New block: block.link = block = self._lookup_label(match["label"]) block.noninstructions.append(line) continue + if match := self.re_global.match(line): + self.globals.add(match["label"]) + block.noninstructions.append(line) + continue if self._re_noninstructions.match(line): if block.instructions: # Non-instruction lines. New block: @@ -152,16 +212,19 @@ def __post_init__(self) -> None: if block.target or not block.fallthrough: # Current block ends with a branch, jump, or return. New block: block.link = block = _Block() - block.instructions.append(line) - if match := self._re_branch.match(line): + inst = self._parse_instruction(line) + block.instructions.append(inst) + if inst.is_branch(): # A block ending in a branch has a target and fallthrough: - block.target = self._lookup_label(match["target"]) + assert inst.target is not None + block.target = self._lookup_label(inst.target) assert block.fallthrough - elif match := self._re_jump.match(line): + elif inst.kind == InstructionKind.JUMP: # A block ending in a jump has a target and no fallthrough: - block.target = self._lookup_label(match["target"]) + assert inst.target is not None + block.target = self._lookup_label(inst.target) block.fallthrough = False - elif self._re_return.match(line): + elif inst.kind == InstructionKind.RETURN: # A block ending in a return has no target and fallthrough: assert not block.target block.fallthrough = False @@ -174,39 +237,47 @@ def _preprocess(self, text: str) -> str: continue_label = f"{self.label_prefix}_JIT_CONTINUE" return re.sub(continue_symbol, continue_label, text) - @classmethod - def _invert_branch(cls, line: str, target: str) -> str | None: - match = cls._re_branch.match(line) - assert match - inverted_reloc = cls._branches.get(match["instruction"]) + def _parse_instruction(self, line: str) -> Instruction: + target = None + if match := self._re_branch.match(line): + target = match["target"] + name = match["instruction"] + if name in self._short_branches: + kind = InstructionKind.SHORT_BRANCH + else: + kind = InstructionKind.LONG_BRANCH + elif match := self._re_jump.match(line): + target = match["target"] + name = line[: -len(target)].strip() + kind = InstructionKind.JUMP + elif match := self._re_return.match(line): + name = line + kind = InstructionKind.RETURN + else: + name, *_ = line.split(" ") + kind = InstructionKind.OTHER + return Instruction(kind, name, line, target) + + def _invert_branch(self, inst: Instruction, target: str) -> Instruction | None: + assert inst.is_branch() + if inst.kind == InstructionKind.SHORT_BRANCH and self._is_far_target(target): + return None + inverted_reloc = self._branches.get(inst.name) if inverted_reloc is None: return None inverted = inverted_reloc[0] if not inverted: return None - (a, b), (c, d) = match.span("instruction"), match.span("target") - # Before: - # je FOO - # After: - # jne BAR - return "".join([line[:a], inverted, line[b:c], target, line[d:]]) - - @classmethod - def _update_jump(cls, line: str, target: str) -> str: - match = cls._re_jump.match(line) - assert match - a, b = match.span("target") - # Before: - # jmp FOO - # After: - # jmp BAR - return "".join([line[:a], target, line[b:]]) + return inst.update_name_and_target(inverted, target) def _lookup_label(self, label: str) -> _Block: if label not in self._labels: self._labels[label] = _Block(label) return self._labels[label] + def _is_far_target(self, label: str) -> bool: + return not label.startswith(self.label_prefix) + def _blocks(self) -> typing.Generator[_Block, None, None]: block: _Block | None = self._root while block: @@ -214,7 +285,7 @@ def _blocks(self) -> typing.Generator[_Block, None, None]: block = block.link def _body(self) -> str: - lines = [] + lines = ["#" + line for line in self.text.splitlines()] hot = True for block in self._blocks(): if hot != block.hot: @@ -222,7 +293,8 @@ def _body(self) -> str: # Make it easy to tell at a glance where cold code is: lines.append(f"# JIT: {'HOT' if hot else 'COLD'} ".ljust(80, "#")) lines.extend(block.noninstructions) - lines.extend(block.instructions) + for inst in block.instructions: + lines.append(inst.text) return "\n".join(lines) def _predecessors(self, block: _Block) -> typing.Generator[_Block, None, None]: @@ -289,8 +361,8 @@ def _invert_hot_branches(self) -> None: if inverted is None: continue branch.instructions[-1] = inverted - jump.instructions[-1] = self._update_jump( - jump.instructions[-1], branch.target.label + jump.instructions[-1] = jump.instructions[-1].update_target( + branch.target.label ) branch.target, jump.target = jump.target, branch.target jump.hot = True @@ -299,49 +371,106 @@ def _remove_redundant_jumps(self) -> None: # Zero-length jumps can be introduced by _insert_continue_label and # _invert_hot_branches: for block in self._blocks(): + target = block.target + if target is None: + continue + target = target.resolve() # Before: # jmp FOO # FOO: # After: # FOO: - if ( - block.target - and block.link - and block.target.resolve() is block.link.resolve() - ): + if block.link and target is block.link.resolve(): block.target = None block.fallthrough = True block.instructions.pop() + # Before: + # br ? FOO: + # ... + # FOO: + # jump BAR + # After: + # br cond BAR + # ... + elif ( + len(target.instructions) == 1 + and target.instructions[0].kind == InstructionKind.JUMP + ): + assert target.target is not None + assert target.target.label is not None + if block.instructions[ + -1 + ].kind == InstructionKind.SHORT_BRANCH and self._is_far_target( + target.target.label + ): + continue + block.target = target.target + block.instructions[-1] = block.instructions[-1].update_target( + target.target.label + ) + + def _find_live_blocks(self) -> set[_Block]: + live: set[_Block] = set() + # Externally reachable blocks are live + todo: set[_Block] = {b for b in self._blocks() if b.label in self.globals} + while todo: + block = todo.pop() + live.add(block) + if block.fallthrough: + next = block.link + if next is not None and next not in live: + todo.add(next) + next = block.target + if next is not None and next not in live: + todo.add(next) + return live + + def _remove_unreachable(self) -> None: + live = self._find_live_blocks() + continuation = self._lookup_label(f"{self.label_prefix}_JIT_CONTINUE") + # Keep blocks after continuation as they may contain data and + # metadata that the assembler needs + prev: _Block | None = None + block = self._root + while block is not continuation: + next = block.link + assert next is not None + if not block in live and prev: + prev.link = next + else: + prev = block + block = next + assert prev.link is block def _fixup_external_labels(self) -> None: if self._supports_external_relocations: # Nothing to fix up return - for block in self._blocks(): + for index, block in enumerate(self._blocks()): if block.target and block.fallthrough: branch = block.instructions[-1] - match = self._re_branch.match(branch) - assert match is not None - target = match["target"] - reloc = self._branches[match["instruction"]][1] - if reloc is not None and not target.startswith(self.label_prefix): + assert branch.is_branch() + target = branch.target + assert target is not None + reloc = self._branches[branch.name][1] + if reloc is not None and self._is_far_target(target): name = target[len(self.symbol_prefix) :] - block.instructions[-1] = ( - f"// target='{target}' prefix='{self.label_prefix}'" - ) - block.instructions.append( - f"{self.symbol_prefix}{reloc}_JIT_RELOCATION_{name}:" + label = f"{self.symbol_prefix}{reloc}_JIT_RELOCATION_{name}_JIT_RELOCATION_{index}:" + block.instructions[-1] = Instruction( + InstructionKind.OTHER, "", label, None ) - a, b = match.span("target") - branch = "".join([branch[:a], "0", branch[b:]]) - block.instructions.append(branch) + block.instructions.append(branch.update_target("0")) def run(self) -> None: """Run this optimizer.""" self._insert_continue_label() self._mark_hot_blocks() - self._invert_hot_branches() - self._remove_redundant_jumps() + # Removing branches can expose opportunities for more branch removal. + # Repeat a few times. 2 would probably do, but it's fast enough with 4. + for _ in range(4): + self._invert_hot_branches() + self._remove_redundant_jumps() + self._remove_unreachable() self._fixup_external_labels() self.path.write_text(self._body()) @@ -350,10 +479,12 @@ class OptimizerAArch64(Optimizer): # pylint: disable = too-few-public-methods """aarch64-pc-windows-msvc/aarch64-apple-darwin/aarch64-unknown-linux-gnu""" _branches = _AARCH64_BRANCHES + _short_branches = _AARCH64_SHORT_BRANCHES # Mach-O does not support the 19 bit branch locations needed for branch reordering _supports_external_relocations = False + _branch_patterns = [name.replace(".", r"\.") for name in _AARCH64_BRANCHES] _re_branch = re.compile( - rf"\s*(?P{'|'.join(_AARCH64_BRANCHES)})\s+(.+,\s+)*(?P[\w.]+)" + rf"\s*(?P{'|'.join(_branch_patterns)})\s+(.+,\s+)*(?P[\w.]+)" ) # https://developer.arm.com/documentation/ddi0602/2025-03/Base-Instructions/B--Branch- @@ -366,6 +497,7 @@ class OptimizerX86(Optimizer): # pylint: disable = too-few-public-methods """i686-pc-windows-msvc/x86_64-apple-darwin/x86_64-unknown-linux-gnu""" _branches = _X86_BRANCHES + _short_branches = {} _re_branch = re.compile( rf"\s*(?P{'|'.join(_X86_BRANCHES)})\s+(?P[\w.]+)" ) diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py index e717365b6b9785..5c45ab930a4ac4 100644 --- a/Tools/jit/_stencils.py +++ b/Tools/jit/_stencils.py @@ -226,7 +226,7 @@ def convert_labels_to_relocations(self) -> None: for name, hole_plus in self.symbols.items(): if isinstance(name, str) and "_JIT_RELOCATION_" in name: _, offset = hole_plus - reloc, target = name.split("_JIT_RELOCATION_") + reloc, target, _ = name.split("_JIT_RELOCATION_") value, symbol = symbol_to_value(target) hole = Hole( int(offset), typing.cast(_schema.HoleKind, reloc), value, symbol, 0 diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 4c188d74a68602..adb8a8d8ecb8a1 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -46,6 +46,7 @@ class _Target(typing.Generic[_S, _R]): optimizer: type[_optimizers.Optimizer] = _optimizers.Optimizer label_prefix: typing.ClassVar[str] symbol_prefix: typing.ClassVar[str] + re_global: typing.ClassVar[re.Pattern[str]] stable: bool = False debug: bool = False verbose: bool = False @@ -180,7 +181,10 @@ async def _compile( "clang", args_s, echo=self.verbose, llvm_version=self.llvm_version ) self.optimizer( - s, label_prefix=self.label_prefix, symbol_prefix=self.symbol_prefix + s, + label_prefix=self.label_prefix, + symbol_prefix=self.symbol_prefix, + re_global=self.re_global, ).run() args_o = [f"--target={self.triple}", "-c", "-o", f"{o}", f"{s}"] await _llvm.run( @@ -355,12 +359,14 @@ class _COFF32(_COFF): # These mangle like Mach-O and other "older" formats: label_prefix = "L" symbol_prefix = "_" + re_global = re.compile(r'\s*\.def\s+(?P
diff --git a/Lib/profiling/sampling/cli.py b/Lib/profiling/sampling/cli.py index 22bfce8c2ead99..3a0444db4c3636 100644 --- a/Lib/profiling/sampling/cli.py +++ b/Lib/profiling/sampling/cli.py @@ -16,6 +16,7 @@ PROFILING_MODE_WALL, PROFILING_MODE_CPU, PROFILING_MODE_GIL, + PROFILING_MODE_EXCEPTION, SORT_MODE_NSAMPLES, SORT_MODE_TOTTIME, SORT_MODE_CUMTIME, @@ -90,6 +91,7 @@ def _parse_mode(mode_string): "wall": PROFILING_MODE_WALL, "cpu": PROFILING_MODE_CPU, "gil": PROFILING_MODE_GIL, + "exception": PROFILING_MODE_EXCEPTION, } return mode_map[mode_string] @@ -213,10 +215,12 @@ def _add_mode_options(parser): mode_group = parser.add_argument_group("Mode options") mode_group.add_argument( "--mode", - choices=["wall", "cpu", "gil"], + choices=["wall", "cpu", "gil", "exception"], default="wall", help="Sampling mode: wall (all samples), cpu (only samples when thread is on CPU), " - "gil (only samples when thread holds the GIL). Incompatible with --async-aware", + "gil (only samples when thread holds the GIL), " + "exception (only samples when thread has an active exception). " + "Incompatible with --async-aware", ) mode_group.add_argument( "--async-mode", diff --git a/Lib/profiling/sampling/collector.py b/Lib/profiling/sampling/collector.py index 22055cf84007b6..a1f6ec190f6556 100644 --- a/Lib/profiling/sampling/collector.py +++ b/Lib/profiling/sampling/collector.py @@ -5,6 +5,7 @@ THREAD_STATUS_ON_CPU, THREAD_STATUS_GIL_REQUESTED, THREAD_STATUS_UNKNOWN, + THREAD_STATUS_HAS_EXCEPTION, ) try: @@ -170,7 +171,7 @@ def _collect_thread_status_stats(self, stack_frames): Returns: tuple: (aggregate_status_counts, has_gc_frame, per_thread_stats) - - aggregate_status_counts: dict with has_gil, on_cpu, etc. + - aggregate_status_counts: dict with has_gil, on_cpu, has_exception, etc. - has_gc_frame: bool indicating if any thread has GC frames - per_thread_stats: dict mapping thread_id to per-thread counts """ @@ -179,6 +180,7 @@ def _collect_thread_status_stats(self, stack_frames): "on_cpu": 0, "gil_requested": 0, "unknown": 0, + "has_exception": 0, "total": 0, } has_gc_frame = False @@ -200,6 +202,8 @@ def _collect_thread_status_stats(self, stack_frames): status_counts["gil_requested"] += 1 if status_flags & THREAD_STATUS_UNKNOWN: status_counts["unknown"] += 1 + if status_flags & THREAD_STATUS_HAS_EXCEPTION: + status_counts["has_exception"] += 1 # Track per-thread statistics thread_id = getattr(thread_info, "thread_id", None) @@ -210,6 +214,7 @@ def _collect_thread_status_stats(self, stack_frames): "on_cpu": 0, "gil_requested": 0, "unknown": 0, + "has_exception": 0, "total": 0, "gc_samples": 0, } @@ -225,6 +230,8 @@ def _collect_thread_status_stats(self, stack_frames): thread_stats["gil_requested"] += 1 if status_flags & THREAD_STATUS_UNKNOWN: thread_stats["unknown"] += 1 + if status_flags & THREAD_STATUS_HAS_EXCEPTION: + thread_stats["has_exception"] += 1 # Check for GC frames in this thread frames = getattr(thread_info, "frame_info", None) diff --git a/Lib/profiling/sampling/constants.py b/Lib/profiling/sampling/constants.py index b05f1703c8505f..34b85ba4b3c61d 100644 --- a/Lib/profiling/sampling/constants.py +++ b/Lib/profiling/sampling/constants.py @@ -5,6 +5,7 @@ PROFILING_MODE_CPU = 1 PROFILING_MODE_GIL = 2 PROFILING_MODE_ALL = 3 # Combines GIL + CPU checks +PROFILING_MODE_EXCEPTION = 4 # Only samples when thread has an active exception # Sort mode constants SORT_MODE_NSAMPLES = 0 @@ -25,6 +26,7 @@ THREAD_STATUS_ON_CPU, THREAD_STATUS_UNKNOWN, THREAD_STATUS_GIL_REQUESTED, + THREAD_STATUS_HAS_EXCEPTION, ) except ImportError: # Fallback for tests or when module is not available @@ -32,3 +34,4 @@ THREAD_STATUS_ON_CPU = (1 << 1) THREAD_STATUS_UNKNOWN = (1 << 2) THREAD_STATUS_GIL_REQUESTED = (1 << 3) + THREAD_STATUS_HAS_EXCEPTION = (1 << 4) diff --git a/Lib/profiling/sampling/gecko_collector.py b/Lib/profiling/sampling/gecko_collector.py index b25ee079dd6ce9..608a15da483729 100644 --- a/Lib/profiling/sampling/gecko_collector.py +++ b/Lib/profiling/sampling/gecko_collector.py @@ -9,13 +9,14 @@ from .collector import Collector from .opcode_utils import get_opcode_info, format_opcode try: - from _remote_debugging import THREAD_STATUS_HAS_GIL, THREAD_STATUS_ON_CPU, THREAD_STATUS_UNKNOWN, THREAD_STATUS_GIL_REQUESTED + from _remote_debugging import THREAD_STATUS_HAS_GIL, THREAD_STATUS_ON_CPU, THREAD_STATUS_UNKNOWN, THREAD_STATUS_GIL_REQUESTED, THREAD_STATUS_HAS_EXCEPTION except ImportError: # Fallback if module not available (shouldn't happen in normal use) THREAD_STATUS_HAS_GIL = (1 << 0) THREAD_STATUS_ON_CPU = (1 << 1) THREAD_STATUS_UNKNOWN = (1 << 2) THREAD_STATUS_GIL_REQUESTED = (1 << 3) + THREAD_STATUS_HAS_EXCEPTION = (1 << 4) # Categories matching Firefox Profiler expectations @@ -28,6 +29,7 @@ {"name": "CPU", "color": "purple", "subcategories": ["Other"]}, {"name": "Code Type", "color": "red", "subcategories": ["Other"]}, {"name": "Opcodes", "color": "magenta", "subcategories": ["Other"]}, + {"name": "Exception", "color": "lightblue", "subcategories": ["Other"]}, ] # Category indices @@ -39,6 +41,7 @@ CATEGORY_CPU = 5 CATEGORY_CODE_TYPE = 6 CATEGORY_OPCODES = 7 +CATEGORY_EXCEPTION = 8 # Subcategory indices DEFAULT_SUBCATEGORY = 0 @@ -88,6 +91,8 @@ def __init__(self, sample_interval_usec, *, skip_idle=False, opcodes=False): self.python_code_start = {} # Thread running Python code (has GIL) self.native_code_start = {} # Thread running native code (on CPU without GIL) self.gil_wait_start = {} # Thread waiting for GIL + self.exception_start = {} # Thread has an exception set + self.no_exception_start = {} # Thread has no exception set # GC event tracking: track GC start time per thread self.gc_start_per_thread = {} # tid -> start_time @@ -204,6 +209,13 @@ def collect(self, stack_frames): self._add_marker(tid, "Waiting for GIL", self.gil_wait_start.pop(tid), current_time, CATEGORY_GIL) + # Track exception state (Has Exception / No Exception) + has_exception = bool(status_flags & THREAD_STATUS_HAS_EXCEPTION) + self._track_state_transition( + tid, has_exception, self.exception_start, self.no_exception_start, + "Has Exception", "No Exception", CATEGORY_EXCEPTION, current_time + ) + # Track GC events by detecting frames in the stack trace # This leverages the improved GC frame tracking from commit 336366fd7ca # which precisely identifies the thread that initiated GC collection @@ -622,6 +634,8 @@ def _finalize_markers(self): (self.native_code_start, "Native Code", CATEGORY_CODE_TYPE), (self.gil_wait_start, "Waiting for GIL", CATEGORY_GIL), (self.gc_start_per_thread, "GC Collecting", CATEGORY_GC), + (self.exception_start, "Has Exception", CATEGORY_EXCEPTION), + (self.no_exception_start, "No Exception", CATEGORY_EXCEPTION), ] for state_dict, marker_name, category in marker_states: diff --git a/Lib/profiling/sampling/live_collector/collector.py b/Lib/profiling/sampling/live_collector/collector.py index 3d25b5969835c0..1652089ad3f52d 100644 --- a/Lib/profiling/sampling/live_collector/collector.py +++ b/Lib/profiling/sampling/live_collector/collector.py @@ -17,6 +17,7 @@ THREAD_STATUS_ON_CPU, THREAD_STATUS_UNKNOWN, THREAD_STATUS_GIL_REQUESTED, + THREAD_STATUS_HAS_EXCEPTION, PROFILING_MODE_CPU, PROFILING_MODE_GIL, PROFILING_MODE_WALL, @@ -61,6 +62,7 @@ class ThreadData: on_cpu: int = 0 gil_requested: int = 0 unknown: int = 0 + has_exception: int = 0 total: int = 0 # Total status samples for this thread # Sample counts @@ -82,6 +84,8 @@ def increment_status_flag(self, status_flags): self.gil_requested += 1 if status_flags & THREAD_STATUS_UNKNOWN: self.unknown += 1 + if status_flags & THREAD_STATUS_HAS_EXCEPTION: + self.has_exception += 1 self.total += 1 def as_status_dict(self): @@ -91,6 +95,7 @@ def as_status_dict(self): "on_cpu": self.on_cpu, "gil_requested": self.gil_requested, "unknown": self.unknown, + "has_exception": self.has_exception, "total": self.total, } @@ -160,6 +165,7 @@ def __init__( "on_cpu": 0, "gil_requested": 0, "unknown": 0, + "has_exception": 0, "total": 0, # Total thread count across all samples } self.gc_frame_samples = 0 # Track samples with GC frames @@ -359,6 +365,7 @@ def collect(self, stack_frames): thread_data.on_cpu += stats.get("on_cpu", 0) thread_data.gil_requested += stats.get("gil_requested", 0) thread_data.unknown += stats.get("unknown", 0) + thread_data.has_exception += stats.get("has_exception", 0) thread_data.total += stats.get("total", 0) if stats.get("gc_samples", 0): thread_data.gc_frame_samples += stats["gc_samples"] @@ -723,6 +730,7 @@ def reset_stats(self): "on_cpu": 0, "gil_requested": 0, "unknown": 0, + "has_exception": 0, "total": 0, } self.gc_frame_samples = 0 diff --git a/Lib/profiling/sampling/live_collector/widgets.py b/Lib/profiling/sampling/live_collector/widgets.py index 869405671ffeed..8f72f69b057628 100644 --- a/Lib/profiling/sampling/live_collector/widgets.py +++ b/Lib/profiling/sampling/live_collector/widgets.py @@ -389,6 +389,7 @@ def draw_thread_status(self, line, width): pct_on_gil = (status_counts["has_gil"] / total_threads) * 100 pct_off_gil = 100.0 - pct_on_gil pct_gil_requested = (status_counts["gil_requested"] / total_threads) * 100 + pct_exception = (status_counts.get("has_exception", 0) / total_threads) * 100 # Get GC percentage based on view mode if thread_data: @@ -427,6 +428,17 @@ def draw_thread_status(self, line, width): add_separator=True, ) + # Show exception stats + if col < width - 15: + col = self._add_percentage_stat( + line, + col, + pct_exception, + "exc", + self.colors["red"], + add_separator=(col > 11), + ) + # Always show GC stats if col < width - 15: col = self._add_percentage_stat( diff --git a/Lib/profiling/sampling/sample.py b/Lib/profiling/sampling/sample.py index d5b8e21134ca18..294ec3003fc6bc 100644 --- a/Lib/profiling/sampling/sample.py +++ b/Lib/profiling/sampling/sample.py @@ -17,6 +17,7 @@ PROFILING_MODE_CPU, PROFILING_MODE_GIL, PROFILING_MODE_ALL, + PROFILING_MODE_EXCEPTION, ) try: from .live_collector import LiveStatsCollector @@ -300,7 +301,8 @@ def sample( all_threads: Whether to sample all threads realtime_stats: Whether to print real-time sampling statistics mode: Profiling mode - WALL (all samples), CPU (only when on CPU), - GIL (only when holding GIL), ALL (includes GIL and CPU status) + GIL (only when holding GIL), ALL (includes GIL and CPU status), + EXCEPTION (only when thread has an active exception) native: Whether to include native frames gc: Whether to include GC frames opcodes: Whether to include opcode information @@ -360,7 +362,8 @@ def sample_live( all_threads: Whether to sample all threads realtime_stats: Whether to print real-time sampling statistics mode: Profiling mode - WALL (all samples), CPU (only when on CPU), - GIL (only when holding GIL), ALL (includes GIL and CPU status) + GIL (only when holding GIL), ALL (includes GIL and CPU status), + EXCEPTION (only when thread has an active exception) native: Whether to include native frames gc: Whether to include GC frames opcodes: Whether to include opcode information diff --git a/Lib/profiling/sampling/stack_collector.py b/Lib/profiling/sampling/stack_collector.py index e5b86719f00b01..b7aa7f5ff82da3 100644 --- a/Lib/profiling/sampling/stack_collector.py +++ b/Lib/profiling/sampling/stack_collector.py @@ -87,12 +87,13 @@ def __init__(self, *args, **kwargs): "on_cpu": 0, "gil_requested": 0, "unknown": 0, + "has_exception": 0, "total": 0, } self.samples_with_gc_frames = 0 # Per-thread statistics - self.per_thread_stats = {} # {thread_id: {has_gil, on_cpu, gil_requested, unknown, total, gc_samples}} + self.per_thread_stats = {} # {thread_id: {has_gil, on_cpu, gil_requested, unknown, has_exception, total, gc_samples}} def collect(self, stack_frames, skip_idle=False): """Override to track thread status statistics before processing frames.""" @@ -118,6 +119,7 @@ def collect(self, stack_frames, skip_idle=False): "on_cpu": 0, "gil_requested": 0, "unknown": 0, + "has_exception": 0, "total": 0, "gc_samples": 0, } @@ -247,12 +249,16 @@ def convert_children(children, min_samples): } # Calculate thread status percentages for display + import sysconfig + is_free_threaded = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) total_threads = max(1, self.thread_status_counts["total"]) thread_stats = { "has_gil_pct": (self.thread_status_counts["has_gil"] / total_threads) * 100, "on_cpu_pct": (self.thread_status_counts["on_cpu"] / total_threads) * 100, "gil_requested_pct": (self.thread_status_counts["gil_requested"] / total_threads) * 100, + "has_exception_pct": (self.thread_status_counts["has_exception"] / total_threads) * 100, "gc_pct": (self.samples_with_gc_frames / max(1, self._sample_count)) * 100, + "free_threaded": is_free_threaded, **self.thread_status_counts } @@ -265,6 +271,7 @@ def convert_children(children, min_samples): "has_gil_pct": (stats["has_gil"] / total) * 100, "on_cpu_pct": (stats["on_cpu"] / total) * 100, "gil_requested_pct": (stats["gil_requested"] / total) * 100, + "has_exception_pct": (stats["has_exception"] / total) * 100, "gc_pct": (stats["gc_samples"] / total_samples_denominator) * 100, **stats } diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 365beec49497a8..4f3beb15f53b33 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -26,11 +26,13 @@ PROFILING_MODE_CPU = 1 PROFILING_MODE_GIL = 2 PROFILING_MODE_ALL = 3 +PROFILING_MODE_EXCEPTION = 4 # Thread status flags THREAD_STATUS_HAS_GIL = 1 << 0 THREAD_STATUS_ON_CPU = 1 << 1 THREAD_STATUS_UNKNOWN = 1 << 2 +THREAD_STATUS_HAS_EXCEPTION = 1 << 4 # Maximum number of retry attempts for operations that may fail transiently MAX_TRIES = 10 @@ -2260,6 +2262,412 @@ def busy_thread(): finally: _cleanup_sockets(*client_sockets, server_socket) + def _make_exception_test_script(self, port): + """Create script with exception and normal threads for testing.""" + return textwrap.dedent( + f"""\ + import socket + import threading + import time + + def exception_thread(): + conn = socket.create_connection(("localhost", {port})) + conn.sendall(b"exception:" + str(threading.get_native_id()).encode()) + try: + raise ValueError("test exception") + except ValueError: + while True: + time.sleep(0.01) + + def normal_thread(): + conn = socket.create_connection(("localhost", {port})) + conn.sendall(b"normal:" + str(threading.get_native_id()).encode()) + while True: + sum(range(1000)) + + t1 = threading.Thread(target=exception_thread) + t2 = threading.Thread(target=normal_thread) + t1.start() + t2.start() + t1.join() + t2.join() + """ + ) + + @contextmanager + def _run_exception_test_process(self): + """Context manager to run exception test script and yield thread IDs and process.""" + port = find_unused_port() + script = self._make_exception_test_script(port) + + with os_helper.temp_dir() as tmp_dir: + script_file = make_script(tmp_dir, "script", script) + server_socket = _create_server_socket(port, backlog=2) + client_sockets = [] + + try: + with _managed_subprocess([sys.executable, script_file]) as p: + exception_tid = None + normal_tid = None + + for _ in range(2): + client_socket, _ = server_socket.accept() + client_sockets.append(client_socket) + line = client_socket.recv(1024) + if line: + if line.startswith(b"exception:"): + try: + exception_tid = int(line.split(b":")[-1]) + except (ValueError, IndexError): + pass + elif line.startswith(b"normal:"): + try: + normal_tid = int(line.split(b":")[-1]) + except (ValueError, IndexError): + pass + + server_socket.close() + server_socket = None + + yield p, exception_tid, normal_tid + finally: + _cleanup_sockets(*client_sockets, server_socket) + + @unittest.skipIf( + sys.platform not in ("linux", "darwin", "win32"), + "Test only runs on supported platforms (Linux, macOS, or Windows)", + ) + @unittest.skipIf( + sys.platform == "android", "Android raises Linux-specific exception" + ) + def test_thread_status_exception_detection(self): + """Test that THREAD_STATUS_HAS_EXCEPTION is set when thread has an active exception.""" + with self._run_exception_test_process() as (p, exception_tid, normal_tid): + self.assertIsNotNone(exception_tid, "Exception thread id not received") + self.assertIsNotNone(normal_tid, "Normal thread id not received") + + statuses = {} + try: + unwinder = RemoteUnwinder( + p.pid, + all_threads=True, + mode=PROFILING_MODE_ALL, + skip_non_matching_threads=False, + ) + for _ in range(MAX_TRIES): + traces = unwinder.get_stack_trace() + statuses = self._get_thread_statuses(traces) + + if ( + exception_tid in statuses + and normal_tid in statuses + and (statuses[exception_tid] & THREAD_STATUS_HAS_EXCEPTION) + and not (statuses[normal_tid] & THREAD_STATUS_HAS_EXCEPTION) + ): + break + time.sleep(0.5) + except PermissionError: + self.skipTest("Insufficient permissions to read the stack trace") + + self.assertIn(exception_tid, statuses) + self.assertIn(normal_tid, statuses) + self.assertTrue( + statuses[exception_tid] & THREAD_STATUS_HAS_EXCEPTION, + "Exception thread should have HAS_EXCEPTION flag", + ) + self.assertFalse( + statuses[normal_tid] & THREAD_STATUS_HAS_EXCEPTION, + "Normal thread should not have HAS_EXCEPTION flag", + ) + + @unittest.skipIf( + sys.platform not in ("linux", "darwin", "win32"), + "Test only runs on supported platforms (Linux, macOS, or Windows)", + ) + @unittest.skipIf( + sys.platform == "android", "Android raises Linux-specific exception" + ) + def test_thread_status_exception_mode_filtering(self): + """Test that PROFILING_MODE_EXCEPTION correctly filters threads.""" + with self._run_exception_test_process() as (p, exception_tid, normal_tid): + self.assertIsNotNone(exception_tid, "Exception thread id not received") + self.assertIsNotNone(normal_tid, "Normal thread id not received") + + try: + unwinder = RemoteUnwinder( + p.pid, + all_threads=True, + mode=PROFILING_MODE_EXCEPTION, + skip_non_matching_threads=True, + ) + for _ in range(MAX_TRIES): + traces = unwinder.get_stack_trace() + statuses = self._get_thread_statuses(traces) + + if exception_tid in statuses: + self.assertNotIn( + normal_tid, + statuses, + "Normal thread should be filtered out in exception mode", + ) + return + time.sleep(0.5) + except PermissionError: + self.skipTest("Insufficient permissions to read the stack trace") + + self.fail("Never found exception thread in exception mode") + +class TestExceptionDetectionScenarios(RemoteInspectionTestBase): + """Test exception detection across all scenarios. + + This class verifies the exact conditions under which THREAD_STATUS_HAS_EXCEPTION + is set. Each test covers a specific scenario: + + 1. except_block: Thread inside except block + -> SHOULD have HAS_EXCEPTION (exc_info->exc_value is set) + + 2. finally_propagating: Exception propagating through finally block + -> SHOULD have HAS_EXCEPTION (current_exception is set) + + 3. finally_after_except: Finally block after except handled exception + -> Should NOT have HAS_EXCEPTION (exc_info cleared after except) + + 4. finally_no_exception: Finally block with no exception raised + -> Should NOT have HAS_EXCEPTION (no exception state) + """ + + def _make_single_scenario_script(self, port, scenario): + """Create script for a single exception scenario.""" + scenarios = { + "except_block": f"""\ +import socket +import threading +import time + +def target_thread(): + '''Inside except block - exception info is present''' + conn = socket.create_connection(("localhost", {port})) + conn.sendall(b"ready:" + str(threading.get_native_id()).encode()) + try: + raise ValueError("test") + except ValueError: + while True: + time.sleep(0.01) + +t = threading.Thread(target=target_thread) +t.start() +t.join() +""", + "finally_propagating": f"""\ +import socket +import threading +import time + +def target_thread(): + '''Exception propagating through finally - current_exception is set''' + conn = socket.create_connection(("localhost", {port})) + conn.sendall(b"ready:" + str(threading.get_native_id()).encode()) + try: + try: + raise ValueError("propagating") + finally: + # Exception is propagating through here + while True: + time.sleep(0.01) + except: + pass # Never reached due to infinite loop + +t = threading.Thread(target=target_thread) +t.start() +t.join() +""", + "finally_after_except": f"""\ +import socket +import threading +import time + +def target_thread(): + '''Finally runs after except handled - exc_info is cleared''' + conn = socket.create_connection(("localhost", {port})) + conn.sendall(b"ready:" + str(threading.get_native_id()).encode()) + try: + raise ValueError("test") + except ValueError: + pass # Exception caught and handled + finally: + while True: + time.sleep(0.01) + +t = threading.Thread(target=target_thread) +t.start() +t.join() +""", + "finally_no_exception": f"""\ +import socket +import threading +import time + +def target_thread(): + '''Finally with no exception at all''' + conn = socket.create_connection(("localhost", {port})) + conn.sendall(b"ready:" + str(threading.get_native_id()).encode()) + try: + pass # No exception + finally: + while True: + time.sleep(0.01) + +t = threading.Thread(target=target_thread) +t.start() +t.join() +""", + } + + return scenarios[scenario] + + @contextmanager + def _run_scenario_process(self, scenario): + """Context manager to run a single scenario and yield thread ID and process.""" + port = find_unused_port() + script = self._make_single_scenario_script(port, scenario) + + with os_helper.temp_dir() as tmp_dir: + script_file = make_script(tmp_dir, "script", script) + server_socket = _create_server_socket(port, backlog=1) + client_socket = None + + try: + with _managed_subprocess([sys.executable, script_file]) as p: + thread_tid = None + + client_socket, _ = server_socket.accept() + line = client_socket.recv(1024) + if line and line.startswith(b"ready:"): + try: + thread_tid = int(line.split(b":")[-1]) + except (ValueError, IndexError): + pass + + server_socket.close() + server_socket = None + + yield p, thread_tid + finally: + _cleanup_sockets(client_socket, server_socket) + + def _check_exception_status(self, p, thread_tid, expect_exception): + """Helper to check if thread has expected exception status.""" + try: + unwinder = RemoteUnwinder( + p.pid, + all_threads=True, + mode=PROFILING_MODE_ALL, + skip_non_matching_threads=False, + ) + + # Collect multiple samples for reliability + results = [] + for _ in range(MAX_TRIES): + traces = unwinder.get_stack_trace() + statuses = self._get_thread_statuses(traces) + + if thread_tid in statuses: + has_exc = bool(statuses[thread_tid] & THREAD_STATUS_HAS_EXCEPTION) + results.append(has_exc) + + if len(results) >= 3: + break + + time.sleep(0.2) + + # Check majority of samples match expected + if not results: + self.fail("Never found target thread in stack traces") + + majority = sum(results) > len(results) // 2 + if expect_exception: + self.assertTrue( + majority, + f"Thread should have HAS_EXCEPTION flag, got {results}" + ) + else: + self.assertFalse( + majority, + f"Thread should NOT have HAS_EXCEPTION flag, got {results}" + ) + + except PermissionError: + self.skipTest("Insufficient permissions to read the stack trace") + + @unittest.skipIf( + sys.platform not in ("linux", "darwin", "win32"), + "Test only runs on supported platforms (Linux, macOS, or Windows)", + ) + @unittest.skipIf( + sys.platform == "android", "Android raises Linux-specific exception" + ) + def test_except_block_has_exception(self): + """Test that thread inside except block has HAS_EXCEPTION flag. + + When a thread is executing inside an except block, exc_info->exc_value + is set, so THREAD_STATUS_HAS_EXCEPTION should be True. + """ + with self._run_scenario_process("except_block") as (p, thread_tid): + self.assertIsNotNone(thread_tid, "Thread ID not received") + self._check_exception_status(p, thread_tid, expect_exception=True) + + @unittest.skipIf( + sys.platform not in ("linux", "darwin", "win32"), + "Test only runs on supported platforms (Linux, macOS, or Windows)", + ) + @unittest.skipIf( + sys.platform == "android", "Android raises Linux-specific exception" + ) + def test_finally_propagating_has_exception(self): + """Test that finally block with propagating exception has HAS_EXCEPTION flag. + + When an exception is propagating through a finally block (not yet caught), + current_exception is set, so THREAD_STATUS_HAS_EXCEPTION should be True. + """ + with self._run_scenario_process("finally_propagating") as (p, thread_tid): + self.assertIsNotNone(thread_tid, "Thread ID not received") + self._check_exception_status(p, thread_tid, expect_exception=True) + + @unittest.skipIf( + sys.platform not in ("linux", "darwin", "win32"), + "Test only runs on supported platforms (Linux, macOS, or Windows)", + ) + @unittest.skipIf( + sys.platform == "android", "Android raises Linux-specific exception" + ) + def test_finally_after_except_no_exception(self): + """Test that finally block after except has NO HAS_EXCEPTION flag. + + When a finally block runs after an except block has handled the exception, + Python clears exc_info before entering finally, so THREAD_STATUS_HAS_EXCEPTION + should be False. + """ + with self._run_scenario_process("finally_after_except") as (p, thread_tid): + self.assertIsNotNone(thread_tid, "Thread ID not received") + self._check_exception_status(p, thread_tid, expect_exception=False) + + @unittest.skipIf( + sys.platform not in ("linux", "darwin", "win32"), + "Test only runs on supported platforms (Linux, macOS, or Windows)", + ) + @unittest.skipIf( + sys.platform == "android", "Android raises Linux-specific exception" + ) + def test_finally_no_exception_no_flag(self): + """Test that finally block with no exception has NO HAS_EXCEPTION flag. + + When a finally block runs during normal execution (no exception raised), + there is no exception state, so THREAD_STATUS_HAS_EXCEPTION should be False. + """ + with self._run_scenario_process("finally_no_exception") as (p, thread_tid): + self.assertIsNotNone(thread_tid, "Thread ID not received") + self._check_exception_status(p, thread_tid, expect_exception=False) + class TestFrameCaching(RemoteInspectionTestBase): """Test that frame caching produces correct results. diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_modes.py b/Lib/test/test_profiling/test_sampling_profiler/test_modes.py index c0457ee7eb8357..c086fbb572b256 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_modes.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_modes.py @@ -427,7 +427,198 @@ def test_parse_mode_function(self): self.assertEqual(_parse_mode("wall"), 0) self.assertEqual(_parse_mode("cpu"), 1) self.assertEqual(_parse_mode("gil"), 2) + self.assertEqual(_parse_mode("exception"), 4) # Test invalid mode raises KeyError with self.assertRaises(KeyError): _parse_mode("invalid") + + +class TestExceptionModeFiltering(unittest.TestCase): + """Test exception mode filtering functionality (--mode=exception).""" + + def test_exception_mode_validation(self): + """Test that CLI accepts exception mode choice correctly.""" + from profiling.sampling.cli import main + + test_args = [ + "profiling.sampling.cli", + "attach", + "12345", + "--mode", + "exception", + ] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.cli.sample") as mock_sample, + ): + try: + main() + except (SystemExit, OSError, RuntimeError): + pass # Expected due to invalid PID + + # Should have attempted to call sample with mode=4 (exception mode) + mock_sample.assert_called_once() + call_args = mock_sample.call_args + # Check the mode parameter (should be in kwargs) + self.assertEqual(call_args.kwargs.get("mode"), 4) # PROFILING_MODE_EXCEPTION + + def test_exception_mode_sample_function_call(self): + """Test that sample() function correctly uses exception mode.""" + with ( + mock.patch( + "profiling.sampling.sample.SampleProfiler" + ) as mock_profiler, + ): + # Mock the profiler instance + mock_instance = mock.Mock() + mock_profiler.return_value = mock_instance + + # Create a real collector instance + collector = PstatsCollector(sample_interval_usec=1000, skip_idle=True) + + # Call sample with exception mode + profiling.sampling.sample.sample( + 12345, + collector, + mode=4, # PROFILING_MODE_EXCEPTION + duration_sec=1, + ) + + # Verify SampleProfiler was created with correct mode + mock_profiler.assert_called_once() + call_args = mock_profiler.call_args + self.assertEqual(call_args[1]["mode"], 4) # mode parameter + + # Verify profiler.sample was called + mock_instance.sample.assert_called_once() + + def test_exception_mode_cli_argument_parsing(self): + """Test CLI argument parsing for exception mode with various options.""" + from profiling.sampling.cli import main + + test_args = [ + "profiling.sampling.cli", + "attach", + "12345", + "--mode", + "exception", + "-i", + "500", + "-d", + "5", + ] + + with ( + mock.patch("sys.argv", test_args), + mock.patch("profiling.sampling.cli.sample") as mock_sample, + ): + try: + main() + except (SystemExit, OSError, RuntimeError): + pass # Expected due to invalid PID + + # Verify all arguments were parsed correctly + mock_sample.assert_called_once() + call_args = mock_sample.call_args + self.assertEqual(call_args.kwargs.get("mode"), 4) # exception mode + self.assertEqual(call_args.kwargs.get("duration_sec"), 5) + + def test_exception_mode_constants_are_defined(self): + """Test that exception mode constant is properly defined.""" + from profiling.sampling.constants import PROFILING_MODE_EXCEPTION + self.assertEqual(PROFILING_MODE_EXCEPTION, 4) + + @requires_subprocess() + def test_exception_mode_integration_filtering(self): + """Integration test: Exception mode should only capture threads with active exceptions.""" + # Script with one thread handling an exception and one normal thread + exception_vs_normal_script = """ +import time +import threading + +exception_ready = threading.Event() + +def normal_worker(): + x = 0 + while True: + x += 1 + +def exception_handling_worker(): + try: + raise ValueError("test exception") + except ValueError: + # Signal AFTER entering except block, then do CPU work + exception_ready.set() + x = 0 + while True: + x += 1 + +normal_thread = threading.Thread(target=normal_worker) +exception_thread = threading.Thread(target=exception_handling_worker) +normal_thread.start() +exception_thread.start() +exception_ready.wait() +_test_sock.sendall(b"working") +normal_thread.join() +exception_thread.join() +""" + with test_subprocess(exception_vs_normal_script, wait_for_working=True) as subproc: + + with ( + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + collector = PstatsCollector(sample_interval_usec=5000, skip_idle=True) + profiling.sampling.sample.sample( + subproc.process.pid, + collector, + duration_sec=2.0, + mode=4, # Exception mode + all_threads=True, + ) + collector.print_stats(show_summary=False, mode=4) + except (PermissionError, RuntimeError) as e: + self.skipTest( + "Insufficient permissions for remote profiling" + ) + + exception_mode_output = captured_output.getvalue() + + # Test wall-clock mode (mode=0) - should capture both functions + with ( + io.StringIO() as captured_output, + mock.patch("sys.stdout", captured_output), + ): + try: + collector = PstatsCollector(sample_interval_usec=5000, skip_idle=False) + profiling.sampling.sample.sample( + subproc.process.pid, + collector, + duration_sec=2.0, + mode=0, # Wall-clock mode + all_threads=True, + ) + collector.print_stats(show_summary=False) + except (PermissionError, RuntimeError) as e: + self.skipTest( + "Insufficient permissions for remote profiling" + ) + + wall_mode_output = captured_output.getvalue() + + # Verify both modes captured samples + self.assertIn("Captured", exception_mode_output) + self.assertIn("samples", exception_mode_output) + self.assertIn("Captured", wall_mode_output) + self.assertIn("samples", wall_mode_output) + + # Exception mode should strongly favor exception_handling_worker over normal_worker + self.assertIn("exception_handling_worker", exception_mode_output) + self.assertNotIn("normal_worker", exception_mode_output) + + # Wall-clock mode should capture both types of work + self.assertIn("exception_handling_worker", wall_mode_output) + self.assertIn("normal_worker", wall_mode_output) diff --git a/Misc/NEWS.d/next/Library/2025-12-11-04-18-49.gh-issue-138122.m3EF9E.rst b/Misc/NEWS.d/next/Library/2025-12-11-04-18-49.gh-issue-138122.m3EF9E.rst new file mode 100644 index 00000000000000..9c471ee438df15 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-11-04-18-49.gh-issue-138122.m3EF9E.rst @@ -0,0 +1,3 @@ +Add ``--mode=exception`` to the sampling profiler to capture samples only from +threads with an active exception, useful for analyzing exception handling +overhead. Patch by Pablo Galindo. diff --git a/Modules/_remote_debugging/_remote_debugging.h b/Modules/_remote_debugging/_remote_debugging.h index 0aa98349296b8a..fcb75b841b742e 100644 --- a/Modules/_remote_debugging/_remote_debugging.h +++ b/Modules/_remote_debugging/_remote_debugging.h @@ -109,10 +109,11 @@ typedef enum _WIN32_THREADSTATE { #define MAX_TLBC_SIZE 2048 /* Thread status flags */ -#define THREAD_STATUS_HAS_GIL (1 << 0) -#define THREAD_STATUS_ON_CPU (1 << 1) -#define THREAD_STATUS_UNKNOWN (1 << 2) -#define THREAD_STATUS_GIL_REQUESTED (1 << 3) +#define THREAD_STATUS_HAS_GIL (1 << 0) +#define THREAD_STATUS_ON_CPU (1 << 1) +#define THREAD_STATUS_UNKNOWN (1 << 2) +#define THREAD_STATUS_GIL_REQUESTED (1 << 3) +#define THREAD_STATUS_HAS_EXCEPTION (1 << 4) /* Exception cause macro */ #define set_exception_cause(unwinder, exc_type, message) \ @@ -209,7 +210,8 @@ enum _ProfilingMode { PROFILING_MODE_WALL = 0, PROFILING_MODE_CPU = 1, PROFILING_MODE_GIL = 2, - PROFILING_MODE_ALL = 3 + PROFILING_MODE_ALL = 3, + PROFILING_MODE_EXCEPTION = 4 }; typedef struct { diff --git a/Modules/_remote_debugging/module.c b/Modules/_remote_debugging/module.c index 9b05b911658190..a194d88c3c3ca0 100644 --- a/Modules/_remote_debugging/module.c +++ b/Modules/_remote_debugging/module.c @@ -568,7 +568,8 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self gc_frame); if (!frame_info) { // Check if this was an intentional skip due to mode-based filtering - if ((self->mode == PROFILING_MODE_CPU || self->mode == PROFILING_MODE_GIL) && !PyErr_Occurred()) { + if ((self->mode == PROFILING_MODE_CPU || self->mode == PROFILING_MODE_GIL || + self->mode == PROFILING_MODE_EXCEPTION) && !PyErr_Occurred()) { // Thread was skipped due to mode filtering, continue to next thread continue; } @@ -1068,6 +1069,9 @@ _remote_debugging_exec(PyObject *m) if (PyModule_AddIntConstant(m, "THREAD_STATUS_GIL_REQUESTED", THREAD_STATUS_GIL_REQUESTED) < 0) { return -1; } + if (PyModule_AddIntConstant(m, "THREAD_STATUS_HAS_EXCEPTION", THREAD_STATUS_HAS_EXCEPTION) < 0) { + return -1; + } if (RemoteDebugging_InitState(st) < 0) { return -1; diff --git a/Modules/_remote_debugging/threads.c b/Modules/_remote_debugging/threads.c index f564e3a7256fa7..81c13ea48e3c49 100644 --- a/Modules/_remote_debugging/threads.c +++ b/Modules/_remote_debugging/threads.c @@ -344,6 +344,33 @@ unwind_stack_for_thread( gil_requested = 0; } + // Check exception state (both raised and handled exceptions) + int has_exception = 0; + + // Check current_exception (exception being raised/propagated) + uintptr_t current_exception = GET_MEMBER(uintptr_t, ts, + unwinder->debug_offsets.thread_state.current_exception); + if (current_exception != 0) { + has_exception = 1; + } + + // Check exc_state.exc_value (exception being handled in except block) + // exc_state is embedded in PyThreadState, so we read it directly from + // the thread state buffer. This catches most cases; nested exception + // handlers where exc_info points elsewhere are rare. + if (!has_exception) { + uintptr_t exc_value = GET_MEMBER(uintptr_t, ts, + unwinder->debug_offsets.thread_state.exc_state + + unwinder->debug_offsets.err_stackitem.exc_value); + if (exc_value != 0) { + has_exception = 1; + } + } + + if (has_exception) { + status_flags |= THREAD_STATUS_HAS_EXCEPTION; + } + // Check CPU status long pthread_id = GET_MEMBER(long, ts, unwinder->debug_offsets.thread_state.thread_id); @@ -368,6 +395,9 @@ unwind_stack_for_thread( } else if (unwinder->mode == PROFILING_MODE_GIL) { // Skip if doesn't have GIL should_skip = !(status_flags & THREAD_STATUS_HAS_GIL); + } else if (unwinder->mode == PROFILING_MODE_EXCEPTION) { + // Skip if thread doesn't have an exception active + should_skip = !(status_flags & THREAD_STATUS_HAS_EXCEPTION); } // PROFILING_MODE_WALL and PROFILING_MODE_ALL never skip } From 9fe6e3ed365f40d89a47c2a255e11f0363e9aa78 Mon Sep 17 00:00:00 2001 From: AZero13 Date: Thu, 11 Dec 2025 16:18:52 -0500 Subject: [PATCH 366/638] gh-142571: Check for errors before calling each syscall in `PyUnstable_CopyPerfMapFile()` (#142460) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Co-authored-by: Victor Stinner Co-authored-by: Pablo Galindo Salgado --- ...-12-11-09-06-36.gh-issue-142571.Csdxnn.rst | 1 + Python/sysmodule.c | 23 ++++++++++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-12-11-09-06-36.gh-issue-142571.Csdxnn.rst diff --git a/Misc/NEWS.d/next/C_API/2025-12-11-09-06-36.gh-issue-142571.Csdxnn.rst b/Misc/NEWS.d/next/C_API/2025-12-11-09-06-36.gh-issue-142571.Csdxnn.rst new file mode 100644 index 00000000000000..ea419b4fe1d6b0 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-12-11-09-06-36.gh-issue-142571.Csdxnn.rst @@ -0,0 +1 @@ +:c:func:`!PyUnstable_CopyPerfMapFile` now checks that opening the file succeeded before flushing. diff --git a/Python/sysmodule.c b/Python/sysmodule.c index b4b441bf4d9519..94eb3164ecad58 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2753,20 +2753,31 @@ PyAPI_FUNC(int) PyUnstable_CopyPerfMapFile(const char* parent_filename) { } char buf[4096]; PyThread_acquire_lock(perf_map_state.map_lock, 1); - int fflush_result = 0, result = 0; + int result = 0; while (1) { size_t bytes_read = fread(buf, 1, sizeof(buf), from); + if (bytes_read == 0) { + if (ferror(from)) { + result = -1; + } + break; + } + size_t bytes_written = fwrite(buf, 1, bytes_read, perf_map_state.perf_map); - fflush_result = fflush(perf_map_state.perf_map); - if (fflush_result != 0 || bytes_read == 0 || bytes_written < bytes_read) { + if (bytes_written < bytes_read) { result = -1; - goto close_and_release; + break; } + + if (fflush(perf_map_state.perf_map) != 0) { + result = -1; + break; + } + if (bytes_read < sizeof(buf) && feof(from)) { - goto close_and_release; + break; } } -close_and_release: fclose(from); PyThread_release_lock(perf_map_state.map_lock); return result; From 0a62f8277e9a0dd9f34b0b070adb83994e81b2a8 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 11 Dec 2025 16:23:19 -0500 Subject: [PATCH 367/638] gh-142534: Avoid TSan warnings in dictobject.c (gh-142544) There are places we use "relaxed" loads where C11 requires "consume" or stronger. Unfortunately, compilers don't really implement "consume" so fake it for our use in a way that avoids upsetting TSan. --- Include/cpython/pyatomic.h | 11 +++++++++++ Include/internal/pycore_pyatomic_ft_wrappers.h | 3 +++ Objects/dictobject.c | 12 ++++++------ 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 2a0c11e7b3ad66..790640309f1e03 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -591,6 +591,17 @@ static inline void _Py_atomic_fence_release(void); // --- aliases --------------------------------------------------------------- +// Compilers don't really support "consume" semantics, so we fake it. Use +// "acquire" with TSan to support false positives. Use "relaxed" otherwise, +// because CPUs on all platforms we support respect address dependencies without +// extra barriers. +// See 2.6.7 in https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2020/p2055r0.pdf +#if defined(_Py_THREAD_SANITIZER) +# define _Py_atomic_load_ptr_consume _Py_atomic_load_ptr_acquire +#else +# define _Py_atomic_load_ptr_consume _Py_atomic_load_ptr_relaxed +#endif + #if SIZEOF_LONG == 8 # define _Py_atomic_load_ulong(p) \ _Py_atomic_load_uint64((uint64_t *)p) diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 2ae0185226f847..817c0763bf899b 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -31,6 +31,8 @@ extern "C" { _Py_atomic_store_ptr(&value, new_value) #define FT_ATOMIC_LOAD_PTR_ACQUIRE(value) \ _Py_atomic_load_ptr_acquire(&value) +#define FT_ATOMIC_LOAD_PTR_CONSUME(value) \ + _Py_atomic_load_ptr_consume(&value) #define FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(value) \ _Py_atomic_load_uintptr_acquire(&value) #define FT_ATOMIC_LOAD_PTR_RELAXED(value) \ @@ -125,6 +127,7 @@ extern "C" { #define FT_ATOMIC_LOAD_SSIZE_ACQUIRE(value) value #define FT_ATOMIC_LOAD_SSIZE_RELAXED(value) value #define FT_ATOMIC_LOAD_PTR_ACQUIRE(value) value +#define FT_ATOMIC_LOAD_PTR_CONSUME(value) value #define FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(value) value #define FT_ATOMIC_LOAD_PTR_RELAXED(value) value #define FT_ATOMIC_LOAD_UINT8(value) value diff --git a/Objects/dictobject.c b/Objects/dictobject.c index e0eef7b46df4b2..ac4a46dab107e8 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -1078,7 +1078,7 @@ compare_unicode_unicode(PyDictObject *mp, PyDictKeysObject *dk, void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) { PyDictUnicodeEntry *ep = &((PyDictUnicodeEntry *)ep0)[ix]; - PyObject *ep_key = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_key); + PyObject *ep_key = FT_ATOMIC_LOAD_PTR_CONSUME(ep->me_key); assert(ep_key != NULL); assert(PyUnicode_CheckExact(ep_key)); if (ep_key == key || @@ -1371,7 +1371,7 @@ compare_unicode_generic_threadsafe(PyDictObject *mp, PyDictKeysObject *dk, void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) { PyDictUnicodeEntry *ep = &((PyDictUnicodeEntry *)ep0)[ix]; - PyObject *startkey = _Py_atomic_load_ptr_relaxed(&ep->me_key); + PyObject *startkey = _Py_atomic_load_ptr_consume(&ep->me_key); assert(startkey == NULL || PyUnicode_CheckExact(ep->me_key)); assert(!PyUnicode_CheckExact(key)); @@ -1414,7 +1414,7 @@ compare_unicode_unicode_threadsafe(PyDictObject *mp, PyDictKeysObject *dk, void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) { PyDictUnicodeEntry *ep = &((PyDictUnicodeEntry *)ep0)[ix]; - PyObject *startkey = _Py_atomic_load_ptr_relaxed(&ep->me_key); + PyObject *startkey = _Py_atomic_load_ptr_consume(&ep->me_key); if (startkey == key) { assert(PyUnicode_CheckExact(startkey)); return 1; @@ -1450,7 +1450,7 @@ compare_generic_threadsafe(PyDictObject *mp, PyDictKeysObject *dk, void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) { PyDictKeyEntry *ep = &((PyDictKeyEntry *)ep0)[ix]; - PyObject *startkey = _Py_atomic_load_ptr_relaxed(&ep->me_key); + PyObject *startkey = _Py_atomic_load_ptr_consume(&ep->me_key); if (startkey == key) { return 1; } @@ -5526,7 +5526,7 @@ dictiter_iternext_threadsafe(PyDictObject *d, PyObject *self, k = _Py_atomic_load_ptr_acquire(&d->ma_keys); assert(i >= 0); if (_PyDict_HasSplitTable(d)) { - PyDictValues *values = _Py_atomic_load_ptr_relaxed(&d->ma_values); + PyDictValues *values = _Py_atomic_load_ptr_consume(&d->ma_values); if (values == NULL) { goto concurrent_modification; } @@ -7114,7 +7114,7 @@ _PyObject_TryGetInstanceAttribute(PyObject *obj, PyObject *name, PyObject **attr Py_BEGIN_CRITICAL_SECTION(dict); if (dict->ma_values == values && FT_ATOMIC_LOAD_UINT8(values->valid)) { - value = _Py_atomic_load_ptr_relaxed(&values->values[ix]); + value = _Py_atomic_load_ptr_consume(&values->values[ix]); *attr = _Py_XNewRefWithLock(value); success = true; } else { From 2eca80ffab5a5fd616a71757a4bf84908bce3a8d Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Thu, 11 Dec 2025 21:28:42 +0000 Subject: [PATCH 368/638] gh-138122: Make Tachyon flamegraph and heatmap output more similar (#142590) --- .../_flamegraph_assets/flamegraph.css | 34 ++++++++++++------- .../sampling/_flamegraph_assets/flamegraph.js | 32 +++++++++++++++++ .../flamegraph_template.html | 11 ++++-- .../sampling/_heatmap_assets/heatmap.css | 18 ---------- .../heatmap_pyfile_template.html | 2 +- .../sampling/_shared_assets/base.css | 4 ++- 6 files changed, 67 insertions(+), 34 deletions(-) diff --git a/Lib/profiling/sampling/_flamegraph_assets/flamegraph.css b/Lib/profiling/sampling/_flamegraph_assets/flamegraph.css index ee699f2982616a..c3b1d955f7f526 100644 --- a/Lib/profiling/sampling/_flamegraph_assets/flamegraph.css +++ b/Lib/profiling/sampling/_flamegraph_assets/flamegraph.css @@ -329,34 +329,44 @@ body.resizing-sidebar { gap: 8px; padding: 8px 10px; background: var(--bg-primary); - border: 1px solid var(--border); + border: 2px solid var(--border); border-radius: 8px; transition: all var(--transition-fast); animation: slideUp 0.4s ease-out backwards; - animation-delay: calc(var(--i, 0) * 0.05s); + animation-delay: calc(var(--i, 0) * 0.08s); overflow: hidden; + position: relative; } -.summary-card:nth-child(1) { --i: 0; } -.summary-card:nth-child(2) { --i: 1; } -.summary-card:nth-child(3) { --i: 2; } -.summary-card:nth-child(4) { --i: 3; } +.summary-card:nth-child(1) { --i: 0; --card-color: 55, 118, 171; } +.summary-card:nth-child(2) { --i: 1; --card-color: 40, 167, 69; } +.summary-card:nth-child(3) { --i: 2; --card-color: 255, 193, 7; } +.summary-card:nth-child(4) { --i: 3; --card-color: 111, 66, 193; } .summary-card:hover { - border-color: var(--accent); - background: var(--accent-glow); + border-color: rgba(var(--card-color), 0.6); + background: linear-gradient(135deg, rgba(var(--card-color), 0.08) 0%, var(--bg-primary) 100%); + transform: translateY(-2px); + box-shadow: 0 4px 12px rgba(var(--card-color), 0.15); } .summary-icon { - font-size: 16px; + font-size: 14px; width: 28px; height: 28px; display: flex; align-items: center; justify-content: center; - background: var(--bg-tertiary); + background: linear-gradient(135deg, rgba(var(--card-color), 0.15) 0%, rgba(var(--card-color), 0.05) 100%); + border: 1px solid rgba(var(--card-color), 0.2); border-radius: 6px; flex-shrink: 0; + transition: all var(--transition-fast); +} + +.summary-card:hover .summary-icon { + transform: scale(1.05); + background: linear-gradient(135deg, rgba(var(--card-color), 0.25) 0%, rgba(var(--card-color), 0.1) 100%); } .summary-data { @@ -368,8 +378,8 @@ body.resizing-sidebar { .summary-value { font-family: var(--font-mono); font-size: 13px; - font-weight: 700; - color: var(--accent); + font-weight: 800; + color: rgb(var(--card-color)); line-height: 1.2; white-space: nowrap; overflow: hidden; diff --git a/Lib/profiling/sampling/_flamegraph_assets/flamegraph.js b/Lib/profiling/sampling/_flamegraph_assets/flamegraph.js index 0370c18a25049f..dc7bfed602f32a 100644 --- a/Lib/profiling/sampling/_flamegraph_assets/flamegraph.js +++ b/Lib/profiling/sampling/_flamegraph_assets/flamegraph.js @@ -187,6 +187,27 @@ function restoreUIState() { } } +// ============================================================================ +// Logo/Favicon Setup +// ============================================================================ + +function setupLogos() { + const logo = document.querySelector('.sidebar-logo-img img'); + if (!logo) return; + + const navbarLogoContainer = document.getElementById('navbar-logo'); + if (navbarLogoContainer) { + const navbarLogo = logo.cloneNode(true); + navbarLogoContainer.appendChild(navbarLogo); + } + + const favicon = document.createElement('link'); + favicon.rel = 'icon'; + favicon.type = 'image/png'; + favicon.href = logo.src; + document.head.appendChild(favicon); +} + // ============================================================================ // Status Bar // ============================================================================ @@ -198,6 +219,11 @@ function updateStatusBar(nodeData, rootValue) { const timeMs = (nodeData.value / 1000).toFixed(2); const percent = rootValue > 0 ? ((nodeData.value / rootValue) * 100).toFixed(1) : "0.0"; + const brandEl = document.getElementById('status-brand'); + const taglineEl = document.getElementById('status-tagline'); + if (brandEl) brandEl.style.display = 'none'; + if (taglineEl) taglineEl.style.display = 'none'; + const locationEl = document.getElementById('status-location'); const funcItem = document.getElementById('status-func-item'); const timeItem = document.getElementById('status-time-item'); @@ -230,6 +256,11 @@ function clearStatusBar() { const el = document.getElementById(id); if (el) el.style.display = 'none'; }); + + const brandEl = document.getElementById('status-brand'); + const taglineEl = document.getElementById('status-tagline'); + if (brandEl) brandEl.style.display = 'flex'; + if (taglineEl) taglineEl.style.display = 'flex'; } // ============================================================================ @@ -1065,6 +1096,7 @@ function exportSVG() { function initFlamegraph() { ensureLibraryLoaded(); restoreUIState(); + setupLogos(); let processedData = EMBEDDED_DATA; if (EMBEDDED_DATA.strings) { diff --git a/Lib/profiling/sampling/_flamegraph_assets/flamegraph_template.html b/Lib/profiling/sampling/_flamegraph_assets/flamegraph_template.html index 29e5fdd3f35069..05277fb225c86f 100644 --- a/Lib/profiling/sampling/_flamegraph_assets/flamegraph_template.html +++ b/Lib/profiling/sampling/_flamegraph_assets/flamegraph_template.html @@ -3,7 +3,7 @@ - Tachyon Profiler - Flamegraph + Tachyon Profiler - Flamegraph Report @@ -15,9 +15,10 @@
+ Tachyon - Profiler + Flamegraph Report
Heat Map
+ + Tachyon Profiler + + + Python Sampling Profiler +
- Back to Index +
-
+
Self Time
Total Time
-
+
Show All
Hot Only
-
+
Heat
Specialization diff --git a/Lib/profiling/sampling/_shared_assets/base.css b/Lib/profiling/sampling/_shared_assets/base.css index 46916709f19f54..c88cf58eef9260 100644 --- a/Lib/profiling/sampling/_shared_assets/base.css +++ b/Lib/profiling/sampling/_shared_assets/base.css @@ -387,6 +387,7 @@ body { button:focus-visible, select:focus-visible, input:focus-visible, +.toggle-switch:focus-visible, a.toolbar-btn:focus-visible { outline: 2px solid var(--python-gold); outline-offset: 2px; From 1356fbed7b1bc0356f0ee3a4bbe140abb25d788d Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Fri, 12 Dec 2025 00:50:17 +0000 Subject: [PATCH 370/638] gh-142374: Fix recursive function cumulative over-counting in sampling profiler (#142378) --- Lib/profiling/sampling/heatmap_collector.py | 22 +- .../sampling/live_collector/collector.py | 47 +++-- .../sampling/live_collector/widgets.py | 20 +- Lib/profiling/sampling/pstats_collector.py | 9 +- .../test_sampling_profiler/test_collectors.py | 199 +++++++++++++++++- .../test_integration.py | 13 +- .../test_live_collector_core.py | 191 ++++++++++++++++- .../test_live_collector_ui.py | 1 + ...-12-07-13-37-18.gh-issue-142374.m3EF9E.rst | 7 + 9 files changed, 454 insertions(+), 55 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-12-07-13-37-18.gh-issue-142374.m3EF9E.rst diff --git a/Lib/profiling/sampling/heatmap_collector.py b/Lib/profiling/sampling/heatmap_collector.py index a860ed870e3e40..45649ce2009bb6 100644 --- a/Lib/profiling/sampling/heatmap_collector.py +++ b/Lib/profiling/sampling/heatmap_collector.py @@ -491,6 +491,10 @@ def __init__(self, *args, **kwargs): # File index (populated during export) self.file_index = {} + # Reusable set for deduplicating line locations within a single sample. + # This avoids over-counting recursive functions in cumulative stats. + self._seen_lines = set() + def set_stats(self, sample_interval_usec, duration_sec, sample_rate, error_rate=None, missed_samples=None, **kwargs): """Set profiling statistics to include in heatmap output. @@ -524,6 +528,7 @@ def process_frames(self, frames, thread_id): thread_id: Thread ID for this stack trace """ self._total_samples += 1 + self._seen_lines.clear() for i, (filename, location, funcname, opcode) in enumerate(frames): # Normalize location to 4-tuple format @@ -533,7 +538,14 @@ def process_frames(self, frames, thread_id): continue # frames[0] is the leaf - where execution is actually happening - self._record_line_sample(filename, lineno, funcname, is_leaf=(i == 0)) + is_leaf = (i == 0) + line_key = (filename, lineno) + count_cumulative = line_key not in self._seen_lines + if count_cumulative: + self._seen_lines.add(line_key) + + self._record_line_sample(filename, lineno, funcname, is_leaf=is_leaf, + count_cumulative=count_cumulative) if opcode is not None: # Set opcodes_enabled flag when we first encounter opcode data @@ -562,11 +574,13 @@ def _is_valid_frame(self, filename, lineno): return True - def _record_line_sample(self, filename, lineno, funcname, is_leaf=False): + def _record_line_sample(self, filename, lineno, funcname, is_leaf=False, + count_cumulative=True): """Record a sample for a specific line.""" # Track cumulative samples (all occurrences in stack) - self.line_samples[(filename, lineno)] += 1 - self.file_samples[filename][lineno] += 1 + if count_cumulative: + self.line_samples[(filename, lineno)] += 1 + self.file_samples[filename][lineno] += 1 # Track self/leaf samples (only when at top of stack) if is_leaf: diff --git a/Lib/profiling/sampling/live_collector/collector.py b/Lib/profiling/sampling/live_collector/collector.py index 1652089ad3f52d..de541a75db61c1 100644 --- a/Lib/profiling/sampling/live_collector/collector.py +++ b/Lib/profiling/sampling/live_collector/collector.py @@ -210,6 +210,8 @@ def __init__( # Trend tracking (initialized after colors are set up) self._trend_tracker = None + self._seen_locations = set() + @property def elapsed_time(self): """Get the elapsed time, frozen when finished.""" @@ -305,15 +307,18 @@ def process_frames(self, frames, thread_id=None): # Get per-thread data if tracking per-thread thread_data = self._get_or_create_thread_data(thread_id) if thread_id is not None else None + self._seen_locations.clear() # Process each frame in the stack to track cumulative calls # frame.location is (lineno, end_lineno, col_offset, end_col_offset), int, or None for frame in frames: lineno = extract_lineno(frame.location) location = (frame.filename, lineno, frame.funcname) - self.result[location]["cumulative_calls"] += 1 - if thread_data: - thread_data.result[location]["cumulative_calls"] += 1 + if location not in self._seen_locations: + self._seen_locations.add(location) + self.result[location]["cumulative_calls"] += 1 + if thread_data: + thread_data.result[location]["cumulative_calls"] += 1 # The top frame gets counted as an inline call (directly executing) top_frame = frames[0] @@ -371,11 +376,13 @@ def collect(self, stack_frames): thread_data.gc_frame_samples += stats["gc_samples"] # Process frames using pre-selected iterator + frames_processed = False for frames, thread_id in self._get_frame_iterator(stack_frames): if not frames: continue self.process_frames(frames, thread_id=thread_id) + frames_processed = True # Track thread IDs if thread_id is not None and thread_id not in self.thread_ids: @@ -388,7 +395,11 @@ def collect(self, stack_frames): if has_gc_frame: self.gc_frame_samples += 1 - self.successful_samples += 1 + # Only count as successful if we actually processed frames + # This is important for modes like --mode exception where most samples + # may be filtered out at the C level + if frames_processed: + self.successful_samples += 1 self.total_samples += 1 # Handle input on every sample for instant responsiveness @@ -659,9 +670,11 @@ def build_stats_list(self): total_time = direct_calls * self.sample_interval_sec cumulative_time = cumulative_calls * self.sample_interval_sec - # Calculate sample percentages - sample_pct = (direct_calls / self.total_samples * 100) if self.total_samples > 0 else 0 - cumul_pct = (cumulative_calls / self.total_samples * 100) if self.total_samples > 0 else 0 + # Calculate sample percentages using successful_samples as denominator + # This ensures percentages are relative to samples that actually had data, + # not all sampling attempts (important for filtered modes like --mode exception) + sample_pct = (direct_calls / self.successful_samples * 100) if self.successful_samples > 0 else 0 + cumul_pct = (cumulative_calls / self.successful_samples * 100) if self.successful_samples > 0 else 0 # Calculate trends for all columns using TrendTracker trends = {} @@ -684,7 +697,9 @@ def build_stats_list(self): "cumulative_calls": cumulative_calls, "total_time": total_time, "cumulative_time": cumulative_time, - "trends": trends, # Dictionary of trends for all columns + "sample_pct": sample_pct, + "cumul_pct": cumul_pct, + "trends": trends, } ) @@ -696,21 +711,9 @@ def build_stats_list(self): elif self.sort_by == "cumtime": stats_list.sort(key=lambda x: x["cumulative_time"], reverse=True) elif self.sort_by == "sample_pct": - stats_list.sort( - key=lambda x: (x["direct_calls"] / self.total_samples * 100) - if self.total_samples > 0 - else 0, - reverse=True, - ) + stats_list.sort(key=lambda x: x["sample_pct"], reverse=True) elif self.sort_by == "cumul_pct": - stats_list.sort( - key=lambda x: ( - x["cumulative_calls"] / self.total_samples * 100 - ) - if self.total_samples > 0 - else 0, - reverse=True, - ) + stats_list.sort(key=lambda x: x["cumul_pct"], reverse=True) return stats_list diff --git a/Lib/profiling/sampling/live_collector/widgets.py b/Lib/profiling/sampling/live_collector/widgets.py index 8f72f69b057628..0ee72119b2faf6 100644 --- a/Lib/profiling/sampling/live_collector/widgets.py +++ b/Lib/profiling/sampling/live_collector/widgets.py @@ -396,6 +396,8 @@ def draw_thread_status(self, line, width): total_samples = max(1, thread_data.sample_count) pct_gc = (thread_data.gc_frame_samples / total_samples) * 100 else: + # Use total_samples for GC percentage since gc_frame_samples is tracked + # across ALL samples (via thread status), not just successful ones total_samples = max(1, self.collector.total_samples) pct_gc = (self.collector.gc_frame_samples / total_samples) * 100 @@ -529,10 +531,7 @@ def draw_top_functions(self, line, width, stats_list): continue func_name = func_data["func"][2] - func_pct = ( - func_data["direct_calls"] - / max(1, self.collector.total_samples) - ) * 100 + func_pct = func_data["sample_pct"] # Medal emoji if col + 3 < width - 15: @@ -765,19 +764,10 @@ def draw_stats_rows(self, line, height, width, stats_list, column_flags): cumulative_calls = stat["cumulative_calls"] total_time = stat["total_time"] cumulative_time = stat["cumulative_time"] + sample_pct = stat["sample_pct"] + cum_pct = stat["cumul_pct"] trends = stat.get("trends", {}) - sample_pct = ( - (direct_calls / self.collector.total_samples * 100) - if self.collector.total_samples > 0 - else 0 - ) - cum_pct = ( - (cumulative_calls / self.collector.total_samples * 100) - if self.collector.total_samples > 0 - else 0 - ) - # Check if this row is selected is_selected = show_opcodes and row_idx == selected_row diff --git a/Lib/profiling/sampling/pstats_collector.py b/Lib/profiling/sampling/pstats_collector.py index 8d787c62bb0677..7c154e25828a8f 100644 --- a/Lib/profiling/sampling/pstats_collector.py +++ b/Lib/profiling/sampling/pstats_collector.py @@ -16,18 +16,23 @@ def __init__(self, sample_interval_usec, *, skip_idle=False): lambda: collections.defaultdict(int) ) self.skip_idle = skip_idle + self._seen_locations = set() def _process_frames(self, frames): """Process a single thread's frame stack.""" if not frames: return + self._seen_locations.clear() + # Process each frame in the stack to track cumulative calls # frame.location is int, tuple (lineno, end_lineno, col_offset, end_col_offset), or None for frame in frames: lineno = extract_lineno(frame.location) - loc = (frame.filename, lineno, frame.funcname) - self.result[loc]["cumulative_calls"] += 1 + location = (frame.filename, lineno, frame.funcname) + if location not in self._seen_locations: + self._seen_locations.add(location) + self.result[location]["cumulative_calls"] += 1 # The top frame gets counted as an inline call (directly executing) top_lineno = extract_lineno(frames[0].location) diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py index 75c4e79591000b..30615a7d31d86c 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py @@ -87,7 +87,7 @@ def test_pstats_collector_with_extreme_intervals_and_empty_data(self): # Should still process the frames self.assertEqual(len(collector.result), 1) - # Test collecting duplicate frames in same sample + # Test collecting duplicate frames in same sample (recursive function) test_frames = [ MockInterpreterInfo( 0, # interpreter_id @@ -96,7 +96,7 @@ def test_pstats_collector_with_extreme_intervals_and_empty_data(self): 1, [ MockFrameInfo("file.py", 10, "func1"), - MockFrameInfo("file.py", 10, "func1"), # Duplicate + MockFrameInfo("file.py", 10, "func1"), # Duplicate (recursion) ], ) ], @@ -104,9 +104,9 @@ def test_pstats_collector_with_extreme_intervals_and_empty_data(self): ] collector = PstatsCollector(sample_interval_usec=1000) collector.collect(test_frames) - # Should count both occurrences + # Should count only once per sample to avoid over-counting recursive functions self.assertEqual( - collector.result[("file.py", 10, "func1")]["cumulative_calls"], 2 + collector.result[("file.py", 10, "func1")]["cumulative_calls"], 1 ) def test_pstats_collector_single_frame_stacks(self): @@ -1205,6 +1205,197 @@ def test_flamegraph_collector_per_thread_gc_percentage(self): self.assertAlmostEqual(per_thread_stats[2]["gc_pct"], 10.0, places=1) +class TestRecursiveFunctionHandling(unittest.TestCase): + """Tests for correct handling of recursive functions in cumulative stats.""" + + def test_pstats_collector_recursive_function_single_sample(self): + """Test that recursive functions are counted once per sample, not per occurrence.""" + collector = PstatsCollector(sample_interval_usec=1000) + + # Simulate a recursive function appearing 5 times in one sample + recursive_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [ + MockFrameInfo("test.py", 10, "recursive_func"), + MockFrameInfo("test.py", 10, "recursive_func"), + MockFrameInfo("test.py", 10, "recursive_func"), + MockFrameInfo("test.py", 10, "recursive_func"), + MockFrameInfo("test.py", 10, "recursive_func"), + ], + ) + ], + ) + ] + collector.collect(recursive_frames) + + location = ("test.py", 10, "recursive_func") + # Should count as 1 cumulative call (present in 1 sample), not 5 + self.assertEqual(collector.result[location]["cumulative_calls"], 1) + # Direct calls should be 1 (top of stack) + self.assertEqual(collector.result[location]["direct_calls"], 1) + + def test_pstats_collector_recursive_function_multiple_samples(self): + """Test cumulative counting across multiple samples with recursion.""" + collector = PstatsCollector(sample_interval_usec=1000) + + # Sample 1: recursive function at depth 3 + sample1 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [ + MockFrameInfo("test.py", 10, "recursive_func"), + MockFrameInfo("test.py", 10, "recursive_func"), + MockFrameInfo("test.py", 10, "recursive_func"), + ], + ) + ], + ) + ] + # Sample 2: recursive function at depth 2 + sample2 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [ + MockFrameInfo("test.py", 10, "recursive_func"), + MockFrameInfo("test.py", 10, "recursive_func"), + ], + ) + ], + ) + ] + # Sample 3: recursive function at depth 4 + sample3 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [ + MockFrameInfo("test.py", 10, "recursive_func"), + MockFrameInfo("test.py", 10, "recursive_func"), + MockFrameInfo("test.py", 10, "recursive_func"), + MockFrameInfo("test.py", 10, "recursive_func"), + ], + ) + ], + ) + ] + + collector.collect(sample1) + collector.collect(sample2) + collector.collect(sample3) + + location = ("test.py", 10, "recursive_func") + # Should count as 3 cumulative calls (present in 3 samples) + # Not 3+2+4=9 which would be the buggy behavior + self.assertEqual(collector.result[location]["cumulative_calls"], 3) + self.assertEqual(collector.result[location]["direct_calls"], 3) + + def test_pstats_collector_mixed_recursive_and_nonrecursive(self): + """Test a call stack with both recursive and non-recursive functions.""" + collector = PstatsCollector(sample_interval_usec=1000) + + # Stack: main -> foo (recursive x3) -> bar + frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [ + MockFrameInfo("test.py", 50, "bar"), # top of stack + MockFrameInfo("test.py", 20, "foo"), # recursive + MockFrameInfo("test.py", 20, "foo"), # recursive + MockFrameInfo("test.py", 20, "foo"), # recursive + MockFrameInfo("test.py", 10, "main"), # bottom + ], + ) + ], + ) + ] + collector.collect(frames) + + # bar: 1 cumulative (in stack), 1 direct (top) + self.assertEqual(collector.result[("test.py", 50, "bar")]["cumulative_calls"], 1) + self.assertEqual(collector.result[("test.py", 50, "bar")]["direct_calls"], 1) + + # foo: 1 cumulative (counted once despite 3 occurrences), 0 direct + self.assertEqual(collector.result[("test.py", 20, "foo")]["cumulative_calls"], 1) + self.assertEqual(collector.result[("test.py", 20, "foo")]["direct_calls"], 0) + + # main: 1 cumulative, 0 direct + self.assertEqual(collector.result[("test.py", 10, "main")]["cumulative_calls"], 1) + self.assertEqual(collector.result[("test.py", 10, "main")]["direct_calls"], 0) + + def test_pstats_collector_cumulative_percentage_cannot_exceed_100(self): + """Test that cumulative percentage stays <= 100% even with deep recursion.""" + collector = PstatsCollector(sample_interval_usec=1000000) # 1 second for easy math + + # Collect 10 samples, each with recursive function at depth 100 + for _ in range(10): + frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [MockFrameInfo("test.py", 10, "deep_recursive")] * 100, + ) + ], + ) + ] + collector.collect(frames) + + location = ("test.py", 10, "deep_recursive") + # Cumulative calls should be 10 (number of samples), not 1000 + self.assertEqual(collector.result[location]["cumulative_calls"], 10) + + # Verify stats calculation gives correct percentage + collector.create_stats() + stats = collector.stats[location] + # stats format: (direct_calls, cumulative_calls, total_time, cumulative_time, callers) + cumulative_calls = stats[1] + self.assertEqual(cumulative_calls, 10) + + def test_pstats_collector_different_lines_same_function_counted_separately(self): + """Test that different line numbers in same function are tracked separately.""" + collector = PstatsCollector(sample_interval_usec=1000) + + # Function with multiple line numbers (e.g., different call sites within recursion) + frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [ + MockFrameInfo("test.py", 15, "func"), # line 15 + MockFrameInfo("test.py", 12, "func"), # line 12 + MockFrameInfo("test.py", 15, "func"), # line 15 again + MockFrameInfo("test.py", 10, "func"), # line 10 + ], + ) + ], + ) + ] + collector.collect(frames) + + # Each unique (file, line, func) should be counted once + self.assertEqual(collector.result[("test.py", 15, "func")]["cumulative_calls"], 1) + self.assertEqual(collector.result[("test.py", 12, "func")]["cumulative_calls"], 1) + self.assertEqual(collector.result[("test.py", 10, "func")]["cumulative_calls"], 1) + + class TestLocationHelpers(unittest.TestCase): """Tests for location handling helper functions.""" diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_integration.py b/Lib/test/test_profiling/test_sampling_profiler/test_integration.py index 029952da697751..b98f1e1191429e 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_integration.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_integration.py @@ -121,16 +121,17 @@ def test_recursive_function_call_counting(self): self.assertIn(fib_key, collector.stats) self.assertIn(main_key, collector.stats) - # Fibonacci should have many calls due to recursion + # Fibonacci: counted once per sample, not per occurrence fib_stats = collector.stats[fib_key] direct_calls, cumulative_calls, tt, ct, callers = fib_stats - # Should have recorded multiple calls (9 total appearances in samples) - self.assertEqual(cumulative_calls, 9) - self.assertGreater(tt, 0) # Should have some total time - self.assertGreater(ct, 0) # Should have some cumulative time + # Should count 3 (present in 3 samples), not 9 (total occurrences) + self.assertEqual(cumulative_calls, 3) + self.assertEqual(direct_calls, 3) # Top of stack in all samples + self.assertGreater(tt, 0) + self.assertGreater(ct, 0) - # Main should have fewer calls + # Main should also have 3 cumulative calls (in all 3 samples) main_stats = collector.stats[main_key] main_direct_calls, main_cumulative_calls = main_stats[0], main_stats[1] self.assertEqual(main_direct_calls, 0) # Never directly executing diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_core.py b/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_core.py index 04e6cd2f1fcb8b..8115ca5528fd65 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_core.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_core.py @@ -157,6 +157,70 @@ def test_process_frames_multiple_threads(self): ) self.assertNotIn(loc1, collector.per_thread_data[456].result) + def test_process_recursive_frames_counted_once(self): + """Test that recursive functions are counted once per sample.""" + collector = LiveStatsCollector(1000) + # Simulate recursive function appearing 5 times in stack + frames = [ + MockFrameInfo("test.py", 10, "recursive_func"), + MockFrameInfo("test.py", 10, "recursive_func"), + MockFrameInfo("test.py", 10, "recursive_func"), + MockFrameInfo("test.py", 10, "recursive_func"), + MockFrameInfo("test.py", 10, "recursive_func"), + ] + collector.process_frames(frames) + + location = ("test.py", 10, "recursive_func") + # Should count as 1 cumulative (present in 1 sample), not 5 + self.assertEqual(collector.result[location]["cumulative_calls"], 1) + self.assertEqual(collector.result[location]["direct_calls"], 1) + + def test_process_recursive_frames_multiple_samples(self): + """Test cumulative counting across multiple samples with recursion.""" + collector = LiveStatsCollector(1000) + + # Sample 1: depth 3 + frames1 = [ + MockFrameInfo("test.py", 10, "recursive_func"), + MockFrameInfo("test.py", 10, "recursive_func"), + MockFrameInfo("test.py", 10, "recursive_func"), + ] + # Sample 2: depth 2 + frames2 = [ + MockFrameInfo("test.py", 10, "recursive_func"), + MockFrameInfo("test.py", 10, "recursive_func"), + ] + + collector.process_frames(frames1) + collector.process_frames(frames2) + + location = ("test.py", 10, "recursive_func") + # Should count as 2 (present in 2 samples), not 5 + self.assertEqual(collector.result[location]["cumulative_calls"], 2) + self.assertEqual(collector.result[location]["direct_calls"], 2) + + def test_process_mixed_recursive_nonrecursive(self): + """Test stack with both recursive and non-recursive functions.""" + collector = LiveStatsCollector(1000) + + # Stack: main -> foo (recursive x3) -> bar + frames = [ + MockFrameInfo("test.py", 50, "bar"), + MockFrameInfo("test.py", 20, "foo"), + MockFrameInfo("test.py", 20, "foo"), + MockFrameInfo("test.py", 20, "foo"), + MockFrameInfo("test.py", 10, "main"), + ] + collector.process_frames(frames) + + # foo: 1 cumulative despite 3 occurrences + self.assertEqual(collector.result[("test.py", 20, "foo")]["cumulative_calls"], 1) + self.assertEqual(collector.result[("test.py", 20, "foo")]["direct_calls"], 0) + + # bar and main: 1 cumulative each + self.assertEqual(collector.result[("test.py", 50, "bar")]["cumulative_calls"], 1) + self.assertEqual(collector.result[("test.py", 10, "main")]["cumulative_calls"], 1) + class TestLiveStatsCollectorCollect(unittest.TestCase): """Tests for the collect method.""" @@ -211,8 +275,11 @@ def test_collect_with_empty_frames(self): collector.collect(stack_frames) - # Empty frames still count as successful since collect() was called successfully - self.assertEqual(collector.successful_samples, 1) + # Empty frames do NOT count as successful - this is important for + # filtered modes like --mode exception where most samples may have + # no matching data. Only samples with actual frame data are counted. + self.assertEqual(collector.successful_samples, 0) + self.assertEqual(collector.total_samples, 1) self.assertEqual(collector.failed_samples, 0) def test_collect_skip_idle_threads(self): @@ -257,6 +324,124 @@ def test_collect_multiple_threads(self): self.assertIn(123, collector.thread_ids) self.assertIn(124, collector.thread_ids) + def test_collect_filtered_mode_percentage_calculation(self): + """Test that percentages use successful_samples, not total_samples. + + This is critical for filtered modes like --mode exception where most + samples may be filtered out at the C level. The percentages should + be relative to samples that actually had frame data, not all attempts. + """ + collector = LiveStatsCollector(1000) + + # Simulate 10 samples where only 2 had matching data (e.g., exception mode) + frames_with_data = [MockFrameInfo("test.py", 10, "exception_handler")] + thread_with_data = MockThreadInfo(123, frames_with_data) + interpreter_with_data = MockInterpreterInfo(0, [thread_with_data]) + + # Empty thread simulates filtered-out data + thread_empty = MockThreadInfo(456, []) + interpreter_empty = MockInterpreterInfo(0, [thread_empty]) + + # 2 samples with data + collector.collect([interpreter_with_data]) + collector.collect([interpreter_with_data]) + + # 8 samples without data (filtered out) + for _ in range(8): + collector.collect([interpreter_empty]) + + # Verify counts + self.assertEqual(collector.total_samples, 10) + self.assertEqual(collector.successful_samples, 2) + + # Build stats and check percentage + stats_list = collector.build_stats_list() + self.assertEqual(len(stats_list), 1) + + # The function appeared in 2 out of 2 successful samples = 100% + # NOT 2 out of 10 total samples = 20% + location = ("test.py", 10, "exception_handler") + self.assertEqual(collector.result[location]["direct_calls"], 2) + + # Verify the percentage calculation in build_stats_list + # direct_calls / successful_samples * 100 = 2/2 * 100 = 100% + # This would be 20% if using total_samples incorrectly + + def test_percentage_values_use_successful_samples(self): + """Test that percentages are calculated from successful_samples. + + This verifies the fix where percentages use successful_samples (samples with + frame data) instead of total_samples (all sampling attempts). Critical for + filtered modes like --mode exception. + """ + collector = LiveStatsCollector(1000) + + # Simulate scenario: 100 total samples, only 20 had frame data + collector.total_samples = 100 + collector.successful_samples = 20 + + # Function appeared in 10 out of 20 successful samples + collector.result[("test.py", 10, "handler")] = { + "direct_calls": 10, + "cumulative_calls": 15, + "total_rec_calls": 0, + } + + stats_list = collector.build_stats_list() + self.assertEqual(len(stats_list), 1) + + stat = stats_list[0] + # Calculate expected percentages using successful_samples + expected_sample_pct = stat["direct_calls"] / collector.successful_samples * 100 + expected_cumul_pct = stat["cumulative_calls"] / collector.successful_samples * 100 + + # Percentage should be 10/20 * 100 = 50%, NOT 10/100 * 100 = 10% + self.assertAlmostEqual(expected_sample_pct, 50.0) + # Cumulative percentage should be 15/20 * 100 = 75%, NOT 15/100 * 100 = 15% + self.assertAlmostEqual(expected_cumul_pct, 75.0) + + # Verify sorting by percentage works correctly + collector.result[("test.py", 20, "other")] = { + "direct_calls": 5, # 25% of successful samples + "cumulative_calls": 8, + "total_rec_calls": 0, + } + collector.sort_by = "sample_pct" + stats_list = collector.build_stats_list() + # handler (50%) should come before other (25%) + self.assertEqual(stats_list[0]["func"][2], "handler") + self.assertEqual(stats_list[1]["func"][2], "other") + + def test_build_stats_list_zero_successful_samples(self): + """Test build_stats_list handles zero successful_samples without division by zero. + + When all samples are filtered out (e.g., exception mode with no exceptions), + percentage calculations should return 0 without raising ZeroDivisionError. + """ + collector = LiveStatsCollector(1000) + + # Edge case: data exists but no successful samples + collector.result[("test.py", 10, "func")] = { + "direct_calls": 10, + "cumulative_calls": 10, + "total_rec_calls": 0, + } + collector.total_samples = 100 + collector.successful_samples = 0 # All samples filtered out + + # Should not raise ZeroDivisionError + stats_list = collector.build_stats_list() + self.assertEqual(len(stats_list), 1) + + # Verify percentage-based sorting also works with zero successful_samples + collector.sort_by = "sample_pct" + stats_list = collector.build_stats_list() + self.assertEqual(len(stats_list), 1) + + collector.sort_by = "cumul_pct" + stats_list = collector.build_stats_list() + self.assertEqual(len(stats_list), 1) + class TestLiveStatsCollectorStatisticsBuilding(unittest.TestCase): """Tests for statistics building and sorting.""" @@ -281,6 +466,8 @@ def setUp(self): "total_rec_calls": 0, } self.collector.total_samples = 300 + # successful_samples is used for percentage calculations + self.collector.successful_samples = 300 def test_build_stats_list(self): """Test that stats list is built correctly.""" diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py b/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py index b5a387fa3a3a71..2ed9d82a4a4aa2 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py @@ -148,6 +148,7 @@ def test_efficiency_bar_visualization(self): def test_stats_display_with_different_sort_modes(self): """Test that stats are displayed correctly with different sort modes.""" self.collector.total_samples = 100 + self.collector.successful_samples = 100 # For percentage calculations self.collector.result[("a.py", 1, "func_a")] = { "direct_calls": 10, "cumulative_calls": 20, diff --git a/Misc/NEWS.d/next/Library/2025-12-07-13-37-18.gh-issue-142374.m3EF9E.rst b/Misc/NEWS.d/next/Library/2025-12-07-13-37-18.gh-issue-142374.m3EF9E.rst new file mode 100644 index 00000000000000..c19100caa36aa5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-07-13-37-18.gh-issue-142374.m3EF9E.rst @@ -0,0 +1,7 @@ +Fix cumulative percentage calculation for recursive functions in the new +sampling profiler. When profiling recursive functions, cumulative statistics +(cumul%, cumtime) could exceed 100% because each recursive frame in a stack +was counted separately. For example, a function recursing 500 times in every +sample would show 50000% cumulative presence. The fix deduplicates locations +within each sample so cumulative stats correctly represent "percentage of +samples where this function was on the stack". Patch by Pablo Galindo. From 3b3838823a3e1baeb9e7568d2271971fc3b9e836 Mon Sep 17 00:00:00 2001 From: ivonastojanovic <80911834+ivonastojanovic@users.noreply.github.com> Date: Fri, 12 Dec 2025 01:36:28 +0000 Subject: [PATCH 371/638] gh-138122: Add inverted flamegraph (#142288) Co-authored-by: Pablo Galindo Salgado --- .../_flamegraph_assets/flamegraph.css | 37 +++ .../sampling/_flamegraph_assets/flamegraph.js | 230 ++++++++++++++++-- .../flamegraph_template.html | 10 + .../sampling/_heatmap_assets/heatmap.css | 90 +------ .../sampling/_shared_assets/base.css | 87 +++++++ ...-12-09-22-11-59.gh-issue-138122.CsoBEo.rst | 8 + 6 files changed, 349 insertions(+), 113 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-12-09-22-11-59.gh-issue-138122.CsoBEo.rst diff --git a/Lib/profiling/sampling/_flamegraph_assets/flamegraph.css b/Lib/profiling/sampling/_flamegraph_assets/flamegraph.css index c3b1d955f7f526..2940f263f7ff29 100644 --- a/Lib/profiling/sampling/_flamegraph_assets/flamegraph.css +++ b/Lib/profiling/sampling/_flamegraph_assets/flamegraph.css @@ -274,6 +274,20 @@ body.resizing-sidebar { flex: 1; } +/* View Mode Section */ +.view-mode-section { + padding-bottom: 20px; + border-bottom: 1px solid var(--border); +} + +.view-mode-section .section-title { + margin-bottom: 12px; +} + +.view-mode-section .toggle-switch { + justify-content: center; +} + /* Collapsible sections */ .collapsible .section-header { display: flex; @@ -986,3 +1000,26 @@ body.resizing-sidebar { grid-template-columns: 1fr; } } + +/* -------------------------------------------------------------------------- + Flamegraph Root Node Styling + -------------------------------------------------------------------------- */ + +/* Style the root node - no border, themed text */ +.d3-flame-graph g:first-of-type rect { + stroke: none; +} + +.d3-flame-graph g:first-of-type .d3-flame-graph-label { + color: var(--text-muted); +} + +/* -------------------------------------------------------------------------- + Flamegraph-Specific Toggle Override + -------------------------------------------------------------------------- */ + +#toggle-invert .toggle-track.on { + background: #8e44ad; + border-color: #8e44ad; + box-shadow: 0 0 8px rgba(142, 68, 173, 0.3); +} diff --git a/Lib/profiling/sampling/_flamegraph_assets/flamegraph.js b/Lib/profiling/sampling/_flamegraph_assets/flamegraph.js index dc7bfed602f32a..fb81094521815e 100644 --- a/Lib/profiling/sampling/_flamegraph_assets/flamegraph.js +++ b/Lib/profiling/sampling/_flamegraph_assets/flamegraph.js @@ -2,8 +2,10 @@ const EMBEDDED_DATA = {{FLAMEGRAPH_DATA}}; // Global string table for resolving string indices let stringTable = []; -let originalData = null; +let normalData = null; +let invertedData = null; let currentThreadFilter = 'all'; +let isInverted = false; // Heat colors are now defined in CSS variables (--heat-1 through --heat-8) // and automatically switch with theme changes - no JS color arrays needed! @@ -94,9 +96,10 @@ function toggleTheme() { } // Re-render flamegraph with new theme colors - if (window.flamegraphData && originalData) { - const tooltip = createPythonTooltip(originalData); - const chart = createFlamegraph(tooltip, originalData.value); + if (window.flamegraphData && normalData) { + const currentData = isInverted ? invertedData : normalData; + const tooltip = createPythonTooltip(currentData); + const chart = createFlamegraph(tooltip, currentData.value); renderFlamegraph(chart, window.flamegraphData); } } @@ -485,6 +488,9 @@ function createFlamegraph(tooltip, rootValue) { .tooltip(tooltip) .inverted(true) .setColorMapper(function (d) { + // Root node should be transparent + if (d.depth === 0) return 'transparent'; + const percentage = d.data.value / rootValue; const level = getHeatLevel(percentage); return heatColors[level]; @@ -796,16 +802,35 @@ function populateProfileSummary(data) { if (rateEl) rateEl.textContent = sampleRate > 0 ? formatNumber(Math.round(sampleRate)) : '--'; // Count unique functions - let functionCount = 0; - function countFunctions(node) { + // Use normal (non-inverted) tree structure, but respect thread filtering + const uniqueFunctions = new Set(); + function collectUniqueFunctions(node) { if (!node) return; - functionCount++; - if (node.children) node.children.forEach(countFunctions); + const filename = resolveString(node.filename) || 'unknown'; + const funcname = resolveString(node.funcname) || resolveString(node.name) || 'unknown'; + const lineno = node.lineno || 0; + const key = `${filename}|${lineno}|${funcname}`; + uniqueFunctions.add(key); + if (node.children) node.children.forEach(collectUniqueFunctions); + } + // In inverted mode, use normalData (with thread filter if active) + // In normal mode, use the passed data (already has thread filter applied if any) + let functionCountSource; + if (!normalData) { + functionCountSource = data; + } else if (isInverted) { + if (currentThreadFilter !== 'all') { + functionCountSource = filterDataByThread(normalData, parseInt(currentThreadFilter)); + } else { + functionCountSource = normalData; + } + } else { + functionCountSource = data; } - countFunctions(data); + collectUniqueFunctions(functionCountSource); const functionsEl = document.getElementById('stat-functions'); - if (functionsEl) functionsEl.textContent = formatNumber(functionCount); + if (functionsEl) functionsEl.textContent = formatNumber(uniqueFunctions.size); // Efficiency bar if (errorRate !== undefined && errorRate !== null) { @@ -840,14 +865,31 @@ function populateProfileSummary(data) { // ============================================================================ function populateStats(data) { - const totalSamples = data.value || 0; - // Populate profile summary populateProfileSummary(data); // Populate thread statistics if available populateThreadStats(data); + // For hotspots: use normal (non-inverted) tree structure, but respect thread filtering. + // In inverted view, the tree structure changes but the hottest functions remain the same. + // However, if a thread filter is active, we need to show that thread's hotspots. + let hotspotSource; + if (!normalData) { + hotspotSource = data; + } else if (isInverted) { + // In inverted mode, use normalData (with thread filter if active) + if (currentThreadFilter !== 'all') { + hotspotSource = filterDataByThread(normalData, parseInt(currentThreadFilter)); + } else { + hotspotSource = normalData; + } + } else { + // In normal mode, use the passed data (already has thread filter applied if any) + hotspotSource = data; + } + const totalSamples = hotspotSource.value || 0; + const functionMap = new Map(); function collectFunctions(node) { @@ -905,7 +947,7 @@ function populateStats(data) { } } - collectFunctions(data); + collectFunctions(hotspotSource); const hotSpots = Array.from(functionMap.values()) .filter(f => f.directPercent > 0.5) @@ -997,19 +1039,20 @@ function initThreadFilter(data) { function filterByThread() { const threadFilter = document.getElementById('thread-filter'); - if (!threadFilter || !originalData) return; + if (!threadFilter || !normalData) return; const selectedThread = threadFilter.value; currentThreadFilter = selectedThread; + const baseData = isInverted ? invertedData : normalData; let filteredData; let selectedThreadId = null; if (selectedThread === 'all') { - filteredData = originalData; + filteredData = baseData; } else { selectedThreadId = parseInt(selectedThread, 10); - filteredData = filterDataByThread(originalData, selectedThreadId); + filteredData = filterDataByThread(baseData, selectedThreadId); if (filteredData.strings) { stringTable = filteredData.strings; @@ -1021,7 +1064,7 @@ function filterByThread() { const chart = createFlamegraph(tooltip, filteredData.value); renderFlamegraph(chart, filteredData); - populateThreadStats(originalData, selectedThreadId); + populateThreadStats(baseData, selectedThreadId); } function filterDataByThread(data, threadId) { @@ -1089,6 +1132,137 @@ function exportSVG() { URL.revokeObjectURL(url); } +// ============================================================================ +// Inverted Flamegraph +// ============================================================================ + +// Example: "file.py|10|foo" or "~|0|" for special frames +function getInvertNodeKey(node) { + return `${node.filename || '~'}|${node.lineno || 0}|${node.funcname || node.name}`; +} + +function accumulateInvertedNode(parent, stackFrame, leaf) { + const key = getInvertNodeKey(stackFrame); + + if (!parent.children[key]) { + parent.children[key] = { + name: stackFrame.name, + value: 0, + children: {}, + filename: stackFrame.filename, + lineno: stackFrame.lineno, + funcname: stackFrame.funcname, + source: stackFrame.source, + threads: new Set() + }; + } + + const node = parent.children[key]; + node.value += leaf.value; + if (leaf.threads) { + leaf.threads.forEach(t => node.threads.add(t)); + } + + return node; +} + +function processLeaf(invertedRoot, path, leafNode) { + if (!path || path.length === 0) { + return; + } + + let invertedParent = accumulateInvertedNode(invertedRoot, leafNode, leafNode); + + // Walk backwards through the call stack + for (let i = path.length - 2; i >= 0; i--) { + invertedParent = accumulateInvertedNode(invertedParent, path[i], leafNode); + } +} + +function traverseInvert(path, currentNode, invertedRoot) { + const children = currentNode.children || []; + const childThreads = new Set(children.flatMap(c => c.threads || [])); + const selfThreads = (currentNode.threads || []).filter(t => !childThreads.has(t)); + + if (selfThreads.length > 0) { + processLeaf(invertedRoot, path, { ...currentNode, threads: selfThreads }); + } + + children.forEach(child => traverseInvert(path.concat([child]), child, invertedRoot)); +} + +function convertInvertDictToArray(node) { + if (node.threads instanceof Set) { + node.threads = Array.from(node.threads).sort((a, b) => a - b); + } + + const children = node.children; + if (children && typeof children === 'object' && !Array.isArray(children)) { + node.children = Object.values(children); + node.children.sort((a, b) => b.value - a.value || a.name.localeCompare(b.name)); + node.children.forEach(convertInvertDictToArray); + } + return node; +} + +function generateInvertedFlamegraph(data) { + const invertedRoot = { + name: data.name, + value: data.value, + children: {}, + stats: data.stats, + threads: data.threads + }; + + const children = data.children || []; + if (children.length === 0) { + // Single-frame tree: the root is its own leaf + processLeaf(invertedRoot, [data], data); + } else { + children.forEach(child => traverseInvert([child], child, invertedRoot)); + } + + convertInvertDictToArray(invertedRoot); + return invertedRoot; +} + +function updateToggleUI(toggleId, isOn) { + const toggle = document.getElementById(toggleId); + if (toggle) { + const track = toggle.querySelector('.toggle-track'); + const labels = toggle.querySelectorAll('.toggle-label'); + if (isOn) { + track.classList.add('on'); + labels[0].classList.remove('active'); + labels[1].classList.add('active'); + } else { + track.classList.remove('on'); + labels[0].classList.add('active'); + labels[1].classList.remove('active'); + } + } +} + +function toggleInvert() { + isInverted = !isInverted; + updateToggleUI('toggle-invert', isInverted); + + // Build inverted data on first use + if (isInverted && !invertedData) { + invertedData = generateInvertedFlamegraph(normalData); + } + + let dataToRender = isInverted ? invertedData : normalData; + + if (currentThreadFilter !== 'all') { + dataToRender = filterDataByThread(dataToRender, parseInt(currentThreadFilter)); + } + + const tooltip = createPythonTooltip(dataToRender); + const chart = createFlamegraph(tooltip, dataToRender.value); + renderFlamegraph(chart, dataToRender); +} + // ============================================================================ // Initialization // ============================================================================ @@ -1098,24 +1272,32 @@ function initFlamegraph() { restoreUIState(); setupLogos(); - let processedData = EMBEDDED_DATA; if (EMBEDDED_DATA.strings) { stringTable = EMBEDDED_DATA.strings; - processedData = resolveStringIndices(EMBEDDED_DATA); + normalData = resolveStringIndices(EMBEDDED_DATA); + } else { + normalData = EMBEDDED_DATA; } // Initialize opcode mapping from embedded data initOpcodeMapping(EMBEDDED_DATA); - originalData = processedData; - initThreadFilter(processedData); + // Inverted data will be built on first toggle + invertedData = null; + + initThreadFilter(normalData); - const tooltip = createPythonTooltip(processedData); - const chart = createFlamegraph(tooltip, processedData.value); - renderFlamegraph(chart, processedData); + const tooltip = createPythonTooltip(normalData); + const chart = createFlamegraph(tooltip, normalData.value); + renderFlamegraph(chart, normalData); initSearchHandlers(); initSidebarResize(); handleResize(); + + const toggleInvertBtn = document.getElementById('toggle-invert'); + if (toggleInvertBtn) { + toggleInvertBtn.addEventListener('click', toggleInvert); + } } if (document.readyState === "loading") { diff --git a/Lib/profiling/sampling/_flamegraph_assets/flamegraph_template.html b/Lib/profiling/sampling/_flamegraph_assets/flamegraph_template.html index 05277fb225c86f..211296a708643f 100644 --- a/Lib/profiling/sampling/_flamegraph_assets/flamegraph_template.html +++ b/Lib/profiling/sampling/_flamegraph_assets/flamegraph_template.html @@ -76,6 +76,16 @@
+ + +
diff --git a/Lib/profiling/sampling/_heatmap_assets/heatmap.js b/Lib/profiling/sampling/_heatmap_assets/heatmap.js index 038aa44b3df619..8ac4ef43e53b37 100644 --- a/Lib/profiling/sampling/_heatmap_assets/heatmap.js +++ b/Lib/profiling/sampling/_heatmap_assets/heatmap.js @@ -24,7 +24,8 @@ function toggleTheme() { // Update theme button icon const btn = document.getElementById('theme-btn'); if (btn) { - btn.innerHTML = next === 'dark' ? '☼' : '☾'; // sun or moon + btn.querySelector('.icon-moon').style.display = next === 'dark' ? 'none' : ''; + btn.querySelector('.icon-sun').style.display = next === 'dark' ? '' : 'none'; } applyLineColors(); @@ -39,7 +40,8 @@ function restoreUIState() { document.documentElement.setAttribute('data-theme', savedTheme); const btn = document.getElementById('theme-btn'); if (btn) { - btn.innerHTML = savedTheme === 'dark' ? '☼' : '☾'; + btn.querySelector('.icon-moon').style.display = savedTheme === 'dark' ? 'none' : ''; + btn.querySelector('.icon-sun').style.display = savedTheme === 'dark' ? '' : 'none'; } } } diff --git a/Lib/profiling/sampling/_heatmap_assets/heatmap_index.js b/Lib/profiling/sampling/_heatmap_assets/heatmap_index.js index 4ddacca5173d34..8eb6af0db5335e 100644 --- a/Lib/profiling/sampling/_heatmap_assets/heatmap_index.js +++ b/Lib/profiling/sampling/_heatmap_assets/heatmap_index.js @@ -28,7 +28,8 @@ function toggleTheme() { // Update theme button icon const btn = document.getElementById('theme-btn'); if (btn) { - btn.innerHTML = next === 'dark' ? '☼' : '☾'; // sun or moon + btn.querySelector('.icon-moon').style.display = next === 'dark' ? 'none' : ''; + btn.querySelector('.icon-sun').style.display = next === 'dark' ? '' : 'none'; } applyHeatmapBarColors(); @@ -41,7 +42,8 @@ function restoreUIState() { document.documentElement.setAttribute('data-theme', savedTheme); const btn = document.getElementById('theme-btn'); if (btn) { - btn.innerHTML = savedTheme === 'dark' ? '☼' : '☾'; + btn.querySelector('.icon-moon').style.display = savedTheme === 'dark' ? 'none' : ''; + btn.querySelector('.icon-sun').style.display = savedTheme === 'dark' ? '' : 'none'; } } } diff --git a/Lib/profiling/sampling/_heatmap_assets/heatmap_index_template.html b/Lib/profiling/sampling/_heatmap_assets/heatmap_index_template.html index 98996bdbf5ffb1..3620f8efb8058a 100644 --- a/Lib/profiling/sampling/_heatmap_assets/heatmap_index_template.html +++ b/Lib/profiling/sampling/_heatmap_assets/heatmap_index_template.html @@ -17,12 +17,29 @@ Heatmap Report
+ + + + + + > + + + + +
diff --git a/Lib/profiling/sampling/_heatmap_assets/heatmap_pyfile_template.html b/Lib/profiling/sampling/_heatmap_assets/heatmap_pyfile_template.html index 3fb6d3a6b91dbb..91b629b2628244 100644 --- a/Lib/profiling/sampling/_heatmap_assets/heatmap_pyfile_template.html +++ b/Lib/profiling/sampling/_heatmap_assets/heatmap_pyfile_template.html @@ -17,14 +17,35 @@
- + + + + + + + + + + + > + + + + +
diff --git a/Lib/profiling/sampling/heatmap_collector.py b/Lib/profiling/sampling/heatmap_collector.py index 45649ce2009bb6..5b4c89283be08c 100644 --- a/Lib/profiling/sampling/heatmap_collector.py +++ b/Lib/profiling/sampling/heatmap_collector.py @@ -858,6 +858,7 @@ def _generate_index_html(self, index_path: Path, file_stats: List[FileStats]): "": f"", "": f"", "": self._template_loader.logo_html, + "": f"{sys.version_info.major}.{sys.version_info.minor}", "": str(len(file_stats)), "": f"{self._total_samples:,}", "": f"{self.stats.get('duration_sec', 0):.1f}s", @@ -915,6 +916,7 @@ def _generate_file_html(self, output_path: Path, filename: str, "": f"", "": f"", "": self._template_loader.logo_html, + "": f"{sys.version_info.major}.{sys.version_info.minor}", } html_content = self._template_loader.file_template diff --git a/Lib/profiling/sampling/stack_collector.py b/Lib/profiling/sampling/stack_collector.py index b7aa7f5ff82da3..e437facd8bb94b 100644 --- a/Lib/profiling/sampling/stack_collector.py +++ b/Lib/profiling/sampling/stack_collector.py @@ -5,6 +5,7 @@ import json import linecache import os +import sys from ._css_utils import get_combined_css from .collector import Collector, extract_lineno @@ -393,6 +394,9 @@ def _create_flamegraph_html(self, data): # Let CSS control size; keep markup simple logo_html = f'Tachyon logo' html_template = html_template.replace("", logo_html) + html_template = html_template.replace( + "", f"{sys.version_info.major}.{sys.version_info.minor}" + ) d3_js = d3_path.read_text(encoding="utf-8") fg_css = fg_css_path.read_text(encoding="utf-8") From 15313dd3d74490f570a3c361a4176437a8320af6 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 12 Dec 2025 17:48:43 +0100 Subject: [PATCH 381/638] gh-140550: Correct error message for PyModExport (PEP 793) hook (GH-142583) --- Python/import.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/import.c b/Python/import.c index 4dd247fac27654..2860ae032dfe29 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2003,7 +2003,7 @@ import_run_modexport(PyThreadState *tstate, PyModExportFunction ex0, if (!PyErr_Occurred()) { PyErr_Format( PyExc_SystemError, - "slot export function for module %s failed without setting an exception", + "module export hook for module %R failed without setting an exception", info->name); } return NULL; @@ -2011,7 +2011,7 @@ import_run_modexport(PyThreadState *tstate, PyModExportFunction ex0, if (PyErr_Occurred()) { PyErr_Format( PyExc_SystemError, - "slot export function for module %s raised unreported exception", + "module export hook for module %R raised unreported exception", info->name); } PyObject *result = PyModule_FromSlotsAndSpec(slots, spec); From 6d644e4453a907710e11f6f6b6b8890c197370d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Preng=C3=A8re?= <2138730+alexprengere@users.noreply.github.com> Date: Fri, 12 Dec 2025 17:58:12 +0100 Subject: [PATCH 382/638] gh-141939: Add colors to interpolated values in argparse (#141940) Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Savannah Ostrowski --- Lib/_colorize.py | 2 +- Lib/argparse.py | 8 ++++-- Lib/test/test_argparse.py | 28 ++++++++++++------- ...-11-28-08-25-19.gh-issue-141939.BXPnFj.rst | 1 + 4 files changed, 26 insertions(+), 13 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-28-08-25-19.gh-issue-141939.BXPnFj.rst diff --git a/Lib/_colorize.py b/Lib/_colorize.py index 29d7cc67b6e39d..0b7047620b4556 100644 --- a/Lib/_colorize.py +++ b/Lib/_colorize.py @@ -169,7 +169,7 @@ class Argparse(ThemeSection): label: str = ANSIColors.BOLD_YELLOW action: str = ANSIColors.BOLD_GREEN default: str = ANSIColors.GREY - default_value: str = ANSIColors.YELLOW + interpolated_value: str = ANSIColors.YELLOW reset: str = ANSIColors.RESET error: str = ANSIColors.BOLD_MAGENTA warning: str = ANSIColors.BOLD_YELLOW diff --git a/Lib/argparse.py b/Lib/argparse.py index ed98aa9e974b2a..ee7ebc4696a0f7 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -668,6 +668,10 @@ def _expand_help(self, action): params[name] = value.__name__ if params.get('choices') is not None: params['choices'] = ', '.join(map(str, params['choices'])) + # Before interpolating, wrap the values with color codes + t = self._theme + for name, value in params.items(): + params[name] = f"{t.interpolated_value}{value}{t.reset}" return help_string % params def _iter_indented_subactions(self, action): @@ -749,8 +753,8 @@ def _get_help_string(self, action): default_str = _(" (default: %(default)s)") prefix, suffix = default_str.split("%(default)s") help += ( - f" {t.default}{prefix.lstrip()}" - f"{t.default_value}%(default)s" + f" {t.default}{prefix.lstrip()}{t.reset}" + f"%(default)s" f"{t.default}{suffix}{t.reset}" ) return help diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 24e8ab1c5cacbb..0f93e8ea740770 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -7308,6 +7308,13 @@ def test_argparse_color(self): choices=("Aaaaa", "Bbbbb", "Ccccc", "Ddddd"), help="pick one", ) + parser.add_argument( + "--optional8", + default="A", + metavar="X", + choices=("A", "B", "C"), + help="among %(choices)s, default is %(default)s", + ) parser.add_argument("+f") parser.add_argument("++bar") @@ -7334,7 +7341,7 @@ def test_argparse_color(self): label_b = self.theme.label pos_b = self.theme.action default = self.theme.default - default_value = self.theme.default_value + interp = self.theme.interpolated_value reset = self.theme.reset # Act @@ -7347,8 +7354,8 @@ def test_argparse_color(self): f"""\ {heading}usage: {reset}{prog}PROG{reset} [{short}-h{reset}] [{short}-v{reset} | {short}-q{reset}] [{short}-o{reset}] [{long}--optional2 {label}OPTIONAL2{reset}] [{long}--optional3 {label}{{X,Y,Z}}{reset}] [{long}--optional4 {label}{{X,Y,Z}}{reset}] [{long}--optional5 {label}{{X,Y,Z}}{reset}] [{long}--optional6 {label}{{X,Y,Z}}{reset}] - [{short}-p {label}{{Aaaaa,Bbbbb,Ccccc,Ddddd}}{reset}] [{short}+f {label}F{reset}] [{long}++bar {label}BAR{reset}] [{long}-+baz {label}BAZ{reset}] - [{short}-c {label}COUNT{reset}] + [{short}-p {label}{{Aaaaa,Bbbbb,Ccccc,Ddddd}}{reset}] [{long}--optional8 {label}X{reset}] [{short}+f {label}F{reset}] [{long}++bar {label}BAR{reset}] + [{long}-+baz {label}BAZ{reset}] [{short}-c {label}COUNT{reset}] {pos}x{reset} {pos}y{reset} {pos}this_indeed_is_a_very_long_action_name{reset} {pos}{{sub1,sub2}} ...{reset} Colorful help @@ -7361,17 +7368,18 @@ def test_argparse_color(self): {heading}options:{reset} {short_b}-h{reset}, {long_b}--help{reset} show this help message and exit - {short_b}-v{reset}, {long_b}--verbose{reset} more spam {default}(default: {default_value}False{default}){reset} - {short_b}-q{reset}, {long_b}--quiet{reset} less spam {default}(default: {default_value}False{default}){reset} + {short_b}-v{reset}, {long_b}--verbose{reset} more spam {default}(default: {reset}{interp}False{reset}{default}){reset} + {short_b}-q{reset}, {long_b}--quiet{reset} less spam {default}(default: {reset}{interp}False{reset}{default}){reset} {short_b}-o{reset}, {long_b}--optional1{reset} {long_b}--optional2{reset} {label_b}OPTIONAL2{reset} - pick one {default}(default: {default_value}None{default}){reset} + pick one {default}(default: {reset}{interp}None{reset}{default}){reset} {long_b}--optional3{reset} {label_b}{{X,Y,Z}}{reset} - {long_b}--optional4{reset} {label_b}{{X,Y,Z}}{reset} pick one {default}(default: {default_value}None{default}){reset} - {long_b}--optional5{reset} {label_b}{{X,Y,Z}}{reset} pick one {default}(default: {default_value}None{default}){reset} - {long_b}--optional6{reset} {label_b}{{X,Y,Z}}{reset} pick one {default}(default: {default_value}None{default}){reset} + {long_b}--optional4{reset} {label_b}{{X,Y,Z}}{reset} pick one {default}(default: {reset}{interp}None{reset}{default}){reset} + {long_b}--optional5{reset} {label_b}{{X,Y,Z}}{reset} pick one {default}(default: {reset}{interp}None{reset}{default}){reset} + {long_b}--optional6{reset} {label_b}{{X,Y,Z}}{reset} pick one {default}(default: {reset}{interp}None{reset}{default}){reset} {short_b}-p{reset}, {long_b}--optional7{reset} {label_b}{{Aaaaa,Bbbbb,Ccccc,Ddddd}}{reset} - pick one {default}(default: {default_value}None{default}){reset} + pick one {default}(default: {reset}{interp}None{reset}{default}){reset} + {long_b}--optional8{reset} {label_b}X{reset} among {interp}A, B, C{reset}, default is {interp}A{reset} {short_b}+f{reset} {label_b}F{reset} {long_b}++bar{reset} {label_b}BAR{reset} {long_b}-+baz{reset} {label_b}BAZ{reset} diff --git a/Misc/NEWS.d/next/Library/2025-11-28-08-25-19.gh-issue-141939.BXPnFj.rst b/Misc/NEWS.d/next/Library/2025-11-28-08-25-19.gh-issue-141939.BXPnFj.rst new file mode 100644 index 00000000000000..1015d90c501fd0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-28-08-25-19.gh-issue-141939.BXPnFj.rst @@ -0,0 +1 @@ +Add color to all interpolated values in :mod:`argparse` help, like ``%(default)s`` or ``%(choices)s``. Patch by Alex Prengère. From 40ac3a9343e9653ad5a15b06741e55f67322eeb2 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Fri, 12 Dec 2025 17:27:12 +0000 Subject: [PATCH 383/638] gh-138122: Tachyon Flamegraph: Make toggle keyboard accesible and adjust sidebar collapse CSS (#142638) --- .../_flamegraph_assets/flamegraph.css | 26 +++++++------------ .../sampling/_flamegraph_assets/flamegraph.js | 11 ++++++++ .../flamegraph_template.html | 19 +++++++++----- .../sampling/_shared_assets/base.css | 1 + 4 files changed, 35 insertions(+), 22 deletions(-) diff --git a/Lib/profiling/sampling/_flamegraph_assets/flamegraph.css b/Lib/profiling/sampling/_flamegraph_assets/flamegraph.css index 2940f263f7ff29..03eb2274d23e68 100644 --- a/Lib/profiling/sampling/_flamegraph_assets/flamegraph.css +++ b/Lib/profiling/sampling/_flamegraph_assets/flamegraph.css @@ -275,16 +275,8 @@ body.resizing-sidebar { } /* View Mode Section */ -.view-mode-section { - padding-bottom: 20px; - border-bottom: 1px solid var(--border); -} - -.view-mode-section .section-title { - margin-bottom: 12px; -} - -.view-mode-section .toggle-switch { +.view-mode-section .section-content { + display: flex; justify-content: center; } @@ -316,7 +308,7 @@ body.resizing-sidebar { } .section-content { - transition: max-height var(--transition-normal), opacity var(--transition-normal); + transition: max-height var(--transition-slow) ease-out, opacity var(--transition-normal) ease-out, padding var(--transition-normal) ease-out; max-height: 1000px; opacity: 1; } @@ -324,7 +316,9 @@ body.resizing-sidebar { .collapsible.collapsed .section-content { max-height: 0; opacity: 0; - margin-bottom: -10px; + padding-top: 0; + pointer-events: none; + transition: max-height var(--transition-slow) ease-in, opacity var(--transition-normal) ease-in, padding var(--transition-normal) ease-in; } /* -------------------------------------------------------------------------- @@ -634,10 +628,6 @@ body.resizing-sidebar { Legend -------------------------------------------------------------------------- */ -.legend-section { - margin-top: auto; - padding-top: 12px; -} .legend { display: flex; @@ -1023,3 +1013,7 @@ body.resizing-sidebar { border-color: #8e44ad; box-shadow: 0 0 8px rgba(142, 68, 173, 0.3); } + +.toggle-switch:focus-visible { + border-radius: 4px; +} diff --git a/Lib/profiling/sampling/_flamegraph_assets/flamegraph.js b/Lib/profiling/sampling/_flamegraph_assets/flamegraph.js index 6345320bd2555d..17fd95af859587 100644 --- a/Lib/profiling/sampling/_flamegraph_assets/flamegraph.js +++ b/Lib/profiling/sampling/_flamegraph_assets/flamegraph.js @@ -1302,6 +1302,17 @@ function initFlamegraph() { } } +// Keyboard shortcut: Enter/Space activates toggle switches +document.addEventListener('keydown', function(e) { + if (e.target.tagName === 'INPUT' || e.target.tagName === 'TEXTAREA') { + return; + } + if ((e.key === 'Enter' || e.key === ' ') && e.target.classList.contains('toggle-switch')) { + e.preventDefault(); + e.target.click(); + } +}); + if (document.readyState === "loading") { document.addEventListener("DOMContentLoaded", initFlamegraph); } else { diff --git a/Lib/profiling/sampling/_flamegraph_assets/flamegraph_template.html b/Lib/profiling/sampling/_flamegraph_assets/flamegraph_template.html index 02855563f83f7c..936c9adfc8c519 100644 --- a/Lib/profiling/sampling/_flamegraph_assets/flamegraph_template.html +++ b/Lib/profiling/sampling/_flamegraph_assets/flamegraph_template.html @@ -102,12 +102,19 @@
- @@ -187,26 +187,51 @@

Runtime Stats

-
-
-
--
-
GIL Held
+
+
+
+ GIL Held + -- +
+
+
+
-
-
--
-
GIL Released
+
+
+ GIL Released + -- +
+
+
+
-
-
--
-
Waiting GIL
+
+
+ Waiting GIL + -- +
+
+
+
-
-
--
-
GC
+
+
+ GC + -- +
+
+
+
-
-
--
-
Exception
+
+
+ Exception + -- +
+
+
+
From ea3fd785cbbc422188833358fbc7ff162d8401f2 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Thu, 25 Dec 2025 20:28:13 +0200 Subject: [PATCH 576/638] gh-142927: Tachyon: Fix contrast ratio in top panel (#142936) --- .../sampling/_flamegraph_assets/flamegraph.css | 8 ++++---- .../sampling/_heatmap_assets/heatmap.css | 14 ++++++-------- Lib/profiling/sampling/_shared_assets/base.css | 16 ++++++++++++++++ 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/Lib/profiling/sampling/_flamegraph_assets/flamegraph.css b/Lib/profiling/sampling/_flamegraph_assets/flamegraph.css index e8fda417428104..24e67bedee5242 100644 --- a/Lib/profiling/sampling/_flamegraph_assets/flamegraph.css +++ b/Lib/profiling/sampling/_flamegraph_assets/flamegraph.css @@ -346,10 +346,10 @@ body.resizing-sidebar { position: relative; } -.summary-card:nth-child(1) { --i: 0; --card-color: 55, 118, 171; } -.summary-card:nth-child(2) { --i: 1; --card-color: 40, 167, 69; } -.summary-card:nth-child(3) { --i: 2; --card-color: 255, 193, 7; } -.summary-card:nth-child(4) { --i: 3; --card-color: 111, 66, 193; } +.summary-card:nth-child(1) { --i: 0; --card-color: var(--card-blue); } +.summary-card:nth-child(2) { --i: 1; --card-color: var(--card-green); } +.summary-card:nth-child(3) { --i: 2; --card-color: var(--card-yellow); } +.summary-card:nth-child(4) { --i: 3; --card-color: var(--card-purple); } .summary-card:hover { border-color: rgba(var(--card-color), 0.6); diff --git a/Lib/profiling/sampling/_heatmap_assets/heatmap.css b/Lib/profiling/sampling/_heatmap_assets/heatmap.css index 9999cd6760fd49..8f7f034ba7e596 100644 --- a/Lib/profiling/sampling/_heatmap_assets/heatmap.css +++ b/Lib/profiling/sampling/_heatmap_assets/heatmap.css @@ -69,12 +69,10 @@ overflow: hidden; } -.stat-card:nth-child(1) { --i: 0; --card-color: 55, 118, 171; } -.stat-card:nth-child(2) { --i: 1; --card-color: 40, 167, 69; } -.stat-card:nth-child(3) { --i: 2; --card-color: 255, 193, 7; } -.stat-card:nth-child(4) { --i: 3; --card-color: 111, 66, 193; } -.stat-card:nth-child(5) { --i: 4; --card-color: 220, 53, 69; } -.stat-card:nth-child(6) { --i: 5; --card-color: 23, 162, 184; } +.stat-card:nth-child(1) { --i: 0; --card-color: var(--card-blue); } +.stat-card:nth-child(2) { --i: 1; --card-color: var(--card-green); } +.stat-card:nth-child(3) { --i: 2; --card-color: var(--card-yellow); } +.stat-card:nth-child(4) { --i: 3; --card-color: var(--card-purple); } .stat-card:hover { border-color: rgba(var(--card-color), 0.6); @@ -159,8 +157,8 @@ overflow: hidden; } -.rate-card:nth-child(5) { animation-delay: 0.32s; --rate-color: 220, 53, 69; } -.rate-card:nth-child(6) { animation-delay: 0.40s; --rate-color: 255, 152, 0; } +.rate-card:nth-child(5) { animation-delay: 0.32s; --rate-color: var(--card-red); } +.rate-card:nth-child(6) { animation-delay: 0.40s; --rate-color: var(--card-orange); } .rate-card:hover { border-color: rgba(var(--rate-color), 0.5); diff --git a/Lib/profiling/sampling/_shared_assets/base.css b/Lib/profiling/sampling/_shared_assets/base.css index cb59a0f77c5b63..2164f3f3aa13c7 100644 --- a/Lib/profiling/sampling/_shared_assets/base.css +++ b/Lib/profiling/sampling/_shared_assets/base.css @@ -100,6 +100,14 @@ /* Heatmap span highlighting colors */ --span-hot-base: 255, 100, 50; --span-cold-base: 150, 150, 150; + + /* Summary card colors - optimized for 4.5:1 contrast on light bg */ + --card-blue: 44, 102, 149; + --card-green: 26, 116, 49; + --card-yellow: 134, 100, 4; + --card-purple: 102, 57, 166; + --card-red: 180, 40, 50; + --card-orange: 166, 90, 0; } /* Dark theme */ @@ -162,6 +170,14 @@ /* Heatmap span highlighting colors - dark theme */ --span-hot-base: 255, 107, 53; --span-cold-base: 189, 189, 189; + + /* Summary card colors - optimized for 4.5:1 contrast on dark bg */ + --card-blue: 88, 166, 255; + --card-green: 63, 185, 80; + --card-yellow: 210, 153, 34; + --card-purple: 163, 113, 247; + --card-red: 248, 113, 113; + --card-orange: 251, 146, 60; } /* -------------------------------------------------------------------------- From 888d101445c72c7cf23923e99ed567732f42fb79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20Kiss=20Koll=C3=A1r?= Date: Thu, 25 Dec 2025 19:21:16 +0000 Subject: [PATCH 577/638] gh-138122: Remove default duration for statistical profiling (#143174) Co-authored-by: Pablo Galindo Salgado --- Doc/library/profiling.sampling.rst | 19 +++++++++---------- Lib/profiling/sampling/cli.py | 20 ++++++++++---------- Lib/profiling/sampling/sample.py | 27 ++++++++++++++------------- 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/Doc/library/profiling.sampling.rst b/Doc/library/profiling.sampling.rst index 370bbcd3242526..dae67cca66d9b4 100644 --- a/Doc/library/profiling.sampling.rst +++ b/Doc/library/profiling.sampling.rst @@ -241,8 +241,8 @@ is unaware it is being profiled. When profiling production systems, keep these guidelines in mind: Start with shorter durations (10-30 seconds) to get quick results, then extend -if you need more statistical accuracy. The default 10-second duration is usually -sufficient to identify major hotspots. +if you need more statistical accuracy. By default, profiling runs until the +target process completes, which is usually sufficient to identify major hotspots. If possible, profile during representative load rather than peak traffic. Profiles collected during normal operation are easier to interpret than those @@ -329,7 +329,7 @@ The default configuration works well for most use cases: * - Default for ``--sampling-rate`` / ``-r`` - 1 kHz * - Default for ``--duration`` / ``-d`` - - 10 seconds + - Run to completion * - Default for ``--all-threads`` / ``-a`` - Main thread only * - Default for ``--native`` @@ -363,15 +363,14 @@ cost of slightly higher profiler CPU usage. Lower rates reduce profiler overhead but may miss short-lived functions. For most applications, the default rate provides a good balance between accuracy and overhead. -The :option:`--duration` option (:option:`-d`) sets how long to profile in seconds. The -default is 10 seconds:: +The :option:`--duration` option (:option:`-d`) sets how long to profile in seconds. By +default, profiling continues until the target process exits or is interrupted:: python -m profiling.sampling run -d 60 script.py -Longer durations collect more samples and produce more statistically reliable -results, especially for code paths that execute infrequently. When profiling -a program that runs for a fixed time, you may want to set the duration to -match or exceed the expected runtime. +Specifying a duration is useful when attaching to long-running processes or when +you want to limit profiling to a specific time window. When profiling a script, +the default behavior of running to completion is usually what you want. Thread selection @@ -1394,7 +1393,7 @@ Sampling options .. option:: -d , --duration - Profiling duration in seconds. Default: 10. + Profiling duration in seconds. Default: run to completion. .. option:: -a, --all-threads diff --git a/Lib/profiling/sampling/cli.py b/Lib/profiling/sampling/cli.py index 10341c1570ceca..dd6431a0322bc7 100644 --- a/Lib/profiling/sampling/cli.py +++ b/Lib/profiling/sampling/cli.py @@ -120,8 +120,8 @@ def _build_child_profiler_args(args): # Sampling options hz = MICROSECONDS_PER_SECOND // args.sample_interval_usec child_args.extend(["-r", str(hz)]) - child_args.extend(["-d", str(args.duration)]) - + if args.duration is not None: + child_args.extend(["-d", str(args.duration)]) if args.all_threads: child_args.append("-a") if args.realtime_stats: @@ -356,9 +356,9 @@ def _add_sampling_options(parser): "-d", "--duration", type=int, - default=10, + default=None, metavar="SECONDS", - help="Sampling duration", + help="Sampling duration (default: run to completion)", ) sampling_group.add_argument( "-a", @@ -562,7 +562,7 @@ def _create_collector(format_type, sample_interval_usec, skip_idle, opcodes=Fals if format_type == "binary": if output_file is None: raise ValueError("Binary format requires an output file") - return collector_class(output_file, interval, skip_idle=skip_idle, + return collector_class(output_file, sample_interval_usec, skip_idle=skip_idle, compression=compression) # Gecko format never skips idle (it needs both GIL and CPU data) @@ -643,11 +643,11 @@ def _validate_args(args, parser): return # Warn about blocking mode with aggressive sampling intervals - if args.blocking and args.interval < 100: + if args.blocking and args.sample_interval_usec < 100: print( - f"Warning: --blocking with a {args.interval} µs interval will stop all threads " - f"{1_000_000 // args.interval} times per second. " - "Consider using --interval 1000 or higher to reduce overhead.", + f"Warning: --blocking with a {args.sample_interval_usec} µs interval will stop all threads " + f"{1_000_000 // args.sample_interval_usec} times per second. " + "Consider using --sampling-rate 1khz or lower to reduce overhead.", file=sys.stderr ) @@ -1107,7 +1107,7 @@ def _handle_live_run(args): if process.poll() is None: process.terminate() try: - process.wait(timeout=_PROCESS_KILL_TIMEOUT) + process.wait(timeout=_PROCESS_KILL_TIMEOUT_SEC) except subprocess.TimeoutExpired: process.kill() process.wait() diff --git a/Lib/profiling/sampling/sample.py b/Lib/profiling/sampling/sample.py index 2fe022c85b0b31..5525bffdf5747d 100644 --- a/Lib/profiling/sampling/sample.py +++ b/Lib/profiling/sampling/sample.py @@ -76,18 +76,18 @@ def _new_unwinder(self, native, gc, opcodes, skip_non_matching_threads): ) return unwinder - def sample(self, collector, duration_sec=10, *, async_aware=False): + def sample(self, collector, duration_sec=None, *, async_aware=False): sample_interval_sec = self.sample_interval_usec / 1_000_000 - running_time = 0 num_samples = 0 errors = 0 interrupted = False + running_time_sec = 0 start_time = next_time = time.perf_counter() last_sample_time = start_time realtime_update_interval = 1.0 # Update every second last_realtime_update = start_time try: - while running_time < duration_sec: + while duration_sec is None or running_time_sec < duration_sec: # Check if live collector wants to stop if hasattr(collector, 'running') and not collector.running: break @@ -104,7 +104,7 @@ def sample(self, collector, duration_sec=10, *, async_aware=False): stack_frames = self.unwinder.get_stack_trace() collector.collect(stack_frames) except ProcessLookupError as e: - duration_sec = current_time - start_time + running_time_sec = current_time - start_time break except (RuntimeError, UnicodeDecodeError, MemoryError, OSError): collector.collect_failed_sample() @@ -135,25 +135,25 @@ def sample(self, collector, duration_sec=10, *, async_aware=False): num_samples += 1 next_time += sample_interval_sec - running_time = time.perf_counter() - start_time + running_time_sec = time.perf_counter() - start_time except KeyboardInterrupt: interrupted = True - running_time = time.perf_counter() - start_time + running_time_sec = time.perf_counter() - start_time print("Interrupted by user.") # Clear real-time stats line if it was being displayed if self.realtime_stats and len(self.sample_intervals) > 0: print() # Add newline after real-time stats - sample_rate = num_samples / running_time if running_time > 0 else 0 + sample_rate = num_samples / running_time_sec if running_time_sec > 0 else 0 error_rate = (errors / num_samples) * 100 if num_samples > 0 else 0 - expected_samples = int(duration_sec / sample_interval_sec) + expected_samples = int(running_time_sec / sample_interval_sec) missed_samples = (expected_samples - num_samples) / expected_samples * 100 if expected_samples > 0 else 0 # Don't print stats for live mode (curses is handling display) is_live_mode = LiveStatsCollector is not None and isinstance(collector, LiveStatsCollector) if not is_live_mode: - print(f"Captured {num_samples:n} samples in {fmt(running_time, 2)} seconds") + print(f"Captured {num_samples:n} samples in {fmt(running_time_sec, 2)} seconds") print(f"Sample rate: {fmt(sample_rate, 2)} samples/sec") print(f"Error rate: {fmt(error_rate, 2)}") @@ -166,7 +166,7 @@ def sample(self, collector, duration_sec=10, *, async_aware=False): # Pass stats to flamegraph collector if it's the right type if hasattr(collector, 'set_stats'): - collector.set_stats(self.sample_interval_usec, running_time, sample_rate, error_rate, missed_samples, mode=self.mode) + collector.set_stats(self.sample_interval_usec, running_time_sec, sample_rate, error_rate, missed_samples, mode=self.mode) if num_samples < expected_samples and not is_live_mode and not interrupted: print( @@ -363,7 +363,7 @@ def sample( pid, collector, *, - duration_sec=10, + duration_sec=None, all_threads=False, realtime_stats=False, mode=PROFILING_MODE_WALL, @@ -378,7 +378,8 @@ def sample( Args: pid: Process ID to sample collector: Collector instance to use for gathering samples - duration_sec: How long to sample for (seconds) + duration_sec: How long to sample for (seconds), or None to run until + the process exits or interrupted all_threads: Whether to sample all threads realtime_stats: Whether to print real-time sampling statistics mode: Profiling mode - WALL (all samples), CPU (only when on CPU), @@ -427,7 +428,7 @@ def sample_live( pid, collector, *, - duration_sec=10, + duration_sec=None, all_threads=False, realtime_stats=False, mode=PROFILING_MODE_WALL, From de22e718bb5c57ec4178aa1787d7634cfc649261 Mon Sep 17 00:00:00 2001 From: Yongtao Huang Date: Fri, 26 Dec 2025 19:11:11 +0800 Subject: [PATCH 578/638] Remove redundant pycore_optimizer.h includes (#143184) `pycore_optimizer.h` was included redundantly in Objects/frameobject.c and Python/instrumentation.c. Both includes are unnecessary and can be safely removed. No functional change. Signed-off-by: Yongtao Huang --- Objects/frameobject.c | 1 - Python/instrumentation.c | 1 - 2 files changed, 2 deletions(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index b652973600c17d..3c0b454503be66 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -17,7 +17,6 @@ #include "frameobject.h" // PyFrameLocalsProxyObject #include "opcode.h" // EXTENDED_ARG -#include "pycore_optimizer.h" #include "clinic/frameobject.c.h" diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 9e750433cffa89..28bbe1d82a3b88 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -18,7 +18,6 @@ #include "pycore_tuple.h" // _PyTuple_FromArraySteal() #include "opcode_ids.h" -#include "pycore_optimizer.h" /* Uncomment this to dump debugging output when assertions fail */ From d3d4cf943209d8f27084af621235aa382ba287b1 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Fri, 26 Dec 2025 16:06:48 +0000 Subject: [PATCH 579/638] gh-140739: Fix crashes from corrupted remote memory (#143190) --- ...-12-26-14-51-50.gh-issue-140739.BAbZTo.rst | 2 + Modules/_remote_debugging/_remote_debugging.h | 6 + Modules/_remote_debugging/asyncio.c | 17 ++- Modules/_remote_debugging/code_objects.c | 117 ++++++++++++++---- Modules/_remote_debugging/frames.c | 15 ++- Modules/_remote_debugging/module.c | 9 ++ Modules/_remote_debugging/object_reading.c | 18 ++- 7 files changed, 150 insertions(+), 34 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-12-26-14-51-50.gh-issue-140739.BAbZTo.rst diff --git a/Misc/NEWS.d/next/Library/2025-12-26-14-51-50.gh-issue-140739.BAbZTo.rst b/Misc/NEWS.d/next/Library/2025-12-26-14-51-50.gh-issue-140739.BAbZTo.rst new file mode 100644 index 00000000000000..ae93cbb51114f1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-26-14-51-50.gh-issue-140739.BAbZTo.rst @@ -0,0 +1,2 @@ +Fix several crashes due to reading invalid memory in the new Tachyon +sampling profiler. Patch by Pablo Galindo. diff --git a/Modules/_remote_debugging/_remote_debugging.h b/Modules/_remote_debugging/_remote_debugging.h index 4119d916be7a63..78add74423b608 100644 --- a/Modules/_remote_debugging/_remote_debugging.h +++ b/Modules/_remote_debugging/_remote_debugging.h @@ -140,6 +140,11 @@ typedef enum _WIN32_THREADSTATE { #define SIZEOF_GC_RUNTIME_STATE sizeof(struct _gc_runtime_state) #define SIZEOF_INTERPRETER_STATE sizeof(PyInterpreterState) +/* Maximum sizes for validation to prevent buffer overflows from corrupted data */ +#define MAX_STACK_CHUNK_SIZE (16 * 1024 * 1024) /* 16 MB max for stack chunks */ +#define MAX_LONG_DIGITS 64 /* Allows values up to ~2^1920 */ +#define MAX_SET_TABLE_SIZE (1 << 20) /* 1 million entries max for set iteration */ + #ifndef MAX #define MAX(a, b) ((a) > (b) ? (a) : (b)) #endif @@ -451,6 +456,7 @@ extern PyObject *make_frame_info( extern bool parse_linetable( const uintptr_t addrq, const char* linetable, + Py_ssize_t linetable_size, int firstlineno, LocationInfo* info ); diff --git a/Modules/_remote_debugging/asyncio.c b/Modules/_remote_debugging/asyncio.c index 7f91f16e3a2ce6..3fcc939fd0e876 100644 --- a/Modules/_remote_debugging/asyncio.c +++ b/Modules/_remote_debugging/asyncio.c @@ -112,9 +112,17 @@ iterate_set_entries( } Py_ssize_t num_els = GET_MEMBER(Py_ssize_t, set_object, unwinder->debug_offsets.set_object.used); - Py_ssize_t set_len = GET_MEMBER(Py_ssize_t, set_object, unwinder->debug_offsets.set_object.mask) + 1; + Py_ssize_t mask = GET_MEMBER(Py_ssize_t, set_object, unwinder->debug_offsets.set_object.mask); uintptr_t table_ptr = GET_MEMBER(uintptr_t, set_object, unwinder->debug_offsets.set_object.table); + // Validate mask and num_els to prevent huge loop iterations from garbage data + if (mask < 0 || mask >= MAX_SET_TABLE_SIZE || num_els < 0 || num_els > mask + 1) { + set_exception_cause(unwinder, PyExc_RuntimeError, + "Invalid set object (corrupted remote memory)"); + return -1; + } + Py_ssize_t set_len = mask + 1; + Py_ssize_t i = 0; Py_ssize_t els = 0; while (i < set_len && els < num_els) { @@ -812,14 +820,15 @@ append_awaited_by_for_thread( return -1; } - if (GET_MEMBER(uintptr_t, task_node, unwinder->debug_offsets.llist_node.next) == 0) { + uintptr_t next_node = GET_MEMBER(uintptr_t, task_node, unwinder->debug_offsets.llist_node.next); + if (next_node == 0) { PyErr_SetString(PyExc_RuntimeError, "Invalid linked list structure reading remote memory"); set_exception_cause(unwinder, PyExc_RuntimeError, "NULL pointer in task linked list"); return -1; } - uintptr_t task_addr = (uintptr_t)GET_MEMBER(uintptr_t, task_node, unwinder->debug_offsets.llist_node.next) + uintptr_t task_addr = next_node - (uintptr_t)unwinder->async_debug_offsets.asyncio_task_object.task_node; if (process_single_task_node(unwinder, task_addr, NULL, result) < 0) { @@ -830,7 +839,7 @@ append_awaited_by_for_thread( // Read next node if (_Py_RemoteDebug_PagedReadRemoteMemory( &unwinder->handle, - (uintptr_t)GET_MEMBER(uintptr_t, task_node, unwinder->debug_offsets.llist_node.next), + next_node, sizeof(task_node), task_node) < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read next task node in awaited_by"); diff --git a/Modules/_remote_debugging/code_objects.c b/Modules/_remote_debugging/code_objects.c index ca6ffe7a00af60..9b7b4dc22b873b 100644 --- a/Modules/_remote_debugging/code_objects.c +++ b/Modules/_remote_debugging/code_objects.c @@ -123,44 +123,74 @@ cache_tlbc_array(RemoteUnwinderObject *unwinder, uintptr_t code_addr, uintptr_t * LINE TABLE PARSING FUNCTIONS * ============================================================================ */ +// Inline helper for bounds-checked byte reading (no function call overhead) +static inline int +read_byte(const uint8_t **ptr, const uint8_t *end, uint8_t *out) +{ + if (*ptr >= end) { + return -1; + } + *out = *(*ptr)++; + return 0; +} + static int -scan_varint(const uint8_t **ptr) +scan_varint(const uint8_t **ptr, const uint8_t *end) { - unsigned int read = **ptr; - *ptr = *ptr + 1; + uint8_t read; + if (read_byte(ptr, end, &read) < 0) { + return -1; + } unsigned int val = read & 63; unsigned int shift = 0; while (read & 64) { - read = **ptr; - *ptr = *ptr + 1; + if (read_byte(ptr, end, &read) < 0) { + return -1; + } shift += 6; + // Prevent infinite loop on malformed data (shift overflow) + if (shift > 28) { + return -1; + } val |= (read & 63) << shift; } - return val; + return (int)val; } static int -scan_signed_varint(const uint8_t **ptr) +scan_signed_varint(const uint8_t **ptr, const uint8_t *end) { - unsigned int uval = scan_varint(ptr); + int uval = scan_varint(ptr, end); + if (uval < 0) { + return INT_MIN; // Error sentinel (valid signed varints won't be INT_MIN) + } if (uval & 1) { - return -(int)(uval >> 1); + return -(int)((unsigned int)uval >> 1); } else { - return uval >> 1; + return (int)((unsigned int)uval >> 1); } } bool -parse_linetable(const uintptr_t addrq, const char* linetable, int firstlineno, LocationInfo* info) +parse_linetable(const uintptr_t addrq, const char* linetable, Py_ssize_t linetable_size, + int firstlineno, LocationInfo* info) { + // Reject garbage: zero or negative size + if (linetable_size <= 0) { + return false; + } + const uint8_t* ptr = (const uint8_t*)(linetable); + const uint8_t* end = ptr + linetable_size; uintptr_t addr = 0; int computed_line = firstlineno; // Running accumulator, separate from output + int val; // Temporary for varint results + uint8_t byte; // Temporary for byte reads const size_t MAX_LINETABLE_ENTRIES = 65536; size_t entry_count = 0; - while (*ptr != '\0' && entry_count < MAX_LINETABLE_ENTRIES) { + while (ptr < end && *ptr != '\0' && entry_count < MAX_LINETABLE_ENTRIES) { entry_count++; uint8_t first_byte = *(ptr++); uint8_t code = (first_byte >> 3) & 15; @@ -173,14 +203,34 @@ parse_linetable(const uintptr_t addrq, const char* linetable, int firstlineno, L info->column = info->end_column = -1; break; case PY_CODE_LOCATION_INFO_LONG: - computed_line += scan_signed_varint(&ptr); + val = scan_signed_varint(&ptr, end); + if (val == INT_MIN) { + return false; + } + computed_line += val; info->lineno = computed_line; - info->end_lineno = computed_line + scan_varint(&ptr); - info->column = scan_varint(&ptr) - 1; - info->end_column = scan_varint(&ptr) - 1; + val = scan_varint(&ptr, end); + if (val < 0) { + return false; + } + info->end_lineno = computed_line + val; + val = scan_varint(&ptr, end); + if (val < 0) { + return false; + } + info->column = val - 1; + val = scan_varint(&ptr, end); + if (val < 0) { + return false; + } + info->end_column = val - 1; break; case PY_CODE_LOCATION_INFO_NO_COLUMNS: - computed_line += scan_signed_varint(&ptr); + val = scan_signed_varint(&ptr, end); + if (val == INT_MIN) { + return false; + } + computed_line += val; info->lineno = info->end_lineno = computed_line; info->column = info->end_column = -1; break; @@ -189,17 +239,25 @@ parse_linetable(const uintptr_t addrq, const char* linetable, int firstlineno, L case PY_CODE_LOCATION_INFO_ONE_LINE2: computed_line += code - 10; info->lineno = info->end_lineno = computed_line; - info->column = *(ptr++); - info->end_column = *(ptr++); + if (read_byte(&ptr, end, &byte) < 0) { + return false; + } + info->column = byte; + if (read_byte(&ptr, end, &byte) < 0) { + return false; + } + info->end_column = byte; break; default: { - uint8_t second_byte = *(ptr++); - if ((second_byte & 128) != 0) { + if (read_byte(&ptr, end, &byte) < 0) { + return false; + } + if ((byte & 128) != 0) { return false; } info->lineno = info->end_lineno = computed_line; - info->column = code << 3 | (second_byte >> 4); - info->end_column = info->column + (second_byte & 15); + info->column = code << 3 | (byte >> 4); + info->end_column = info->column + (byte & 15); break; } } @@ -384,8 +442,14 @@ parse_code_object(RemoteUnwinderObject *unwinder, tlbc_entry = get_tlbc_cache_entry(unwinder, real_address, unwinder->tlbc_generation); } - if (tlbc_entry && ctx->tlbc_index < tlbc_entry->tlbc_array_size) { - assert(ctx->tlbc_index >= 0); + // Validate tlbc_index and check TLBC cache + if (tlbc_entry) { + // Validate index bounds (also catches negative values since tlbc_index is signed) + if (ctx->tlbc_index < 0 || ctx->tlbc_index >= tlbc_entry->tlbc_array_size) { + set_exception_cause(unwinder, PyExc_RuntimeError, + "Invalid tlbc_index (corrupted remote memory)"); + goto error; + } assert(tlbc_entry->tlbc_array_size > 0); // Use cached TLBC data uintptr_t *entries = (uintptr_t *)((char *)tlbc_entry->tlbc_array + sizeof(Py_ssize_t)); @@ -398,7 +462,7 @@ parse_code_object(RemoteUnwinderObject *unwinder, } } - // Fall back to main bytecode + // Fall back to main bytecode (no tlbc_entry or tlbc_bytecode_addr was 0) addrq = (uint16_t *)ip - (uint16_t *)meta->addr_code_adaptive; done_tlbc: @@ -409,6 +473,7 @@ parse_code_object(RemoteUnwinderObject *unwinder, ; // Empty statement to avoid C23 extension warning LocationInfo info = {0}; bool ok = parse_linetable(addrq, PyBytes_AS_STRING(meta->linetable), + PyBytes_GET_SIZE(meta->linetable), meta->first_lineno, &info); if (!ok) { info.lineno = -1; diff --git a/Modules/_remote_debugging/frames.c b/Modules/_remote_debugging/frames.c index d9fece6459875a..02c48205b85a37 100644 --- a/Modules/_remote_debugging/frames.c +++ b/Modules/_remote_debugging/frames.c @@ -45,6 +45,15 @@ process_single_stack_chunk( // Check actual size and reread if necessary size_t actual_size = GET_MEMBER(size_t, this_chunk, offsetof(_PyStackChunk, size)); if (actual_size != current_size) { + // Validate size: reject garbage (too small or unreasonably large) + // Size must be at least enough for the header and reasonably bounded + if (actual_size <= offsetof(_PyStackChunk, data) || actual_size > MAX_STACK_CHUNK_SIZE) { + PyMem_RawFree(this_chunk); + set_exception_cause(unwinder, PyExc_RuntimeError, + "Invalid stack chunk size (corrupted remote memory)"); + return -1; + } + this_chunk = PyMem_RawRealloc(this_chunk, actual_size); if (!this_chunk) { PyErr_NoMemory(); @@ -129,7 +138,11 @@ void * find_frame_in_chunks(StackChunkList *chunks, uintptr_t remote_ptr) { for (size_t i = 0; i < chunks->count; ++i) { - assert(chunks->chunks[i].size > offsetof(_PyStackChunk, data)); + // Validate size: reject garbage that would cause underflow + if (chunks->chunks[i].size <= offsetof(_PyStackChunk, data)) { + // Skip this chunk - corrupted size from remote memory + continue; + } uintptr_t base = chunks->chunks[i].remote_addr + offsetof(_PyStackChunk, data); size_t payload = chunks->chunks[i].size - offsetof(_PyStackChunk, data); diff --git a/Modules/_remote_debugging/module.c b/Modules/_remote_debugging/module.c index e14a23357fb400..737787a331f948 100644 --- a/Modules/_remote_debugging/module.c +++ b/Modules/_remote_debugging/module.c @@ -584,6 +584,7 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self } while (current_tstate != 0) { + uintptr_t prev_tstate = current_tstate; PyObject* frame_info = unwind_stack_for_thread(self, ¤t_tstate, gil_holder_tstate, gc_frame); @@ -591,6 +592,14 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self // Check if this was an intentional skip due to mode-based filtering if ((self->mode == PROFILING_MODE_CPU || self->mode == PROFILING_MODE_GIL || self->mode == PROFILING_MODE_EXCEPTION) && !PyErr_Occurred()) { + // Detect cycle: if current_tstate didn't advance, we have corrupted data + if (current_tstate == prev_tstate) { + Py_DECREF(interpreter_threads); + set_exception_cause(self, PyExc_RuntimeError, + "Thread list cycle detected (corrupted remote memory)"); + Py_CLEAR(result); + goto exit; + } // Thread was skipped due to mode filtering, continue to next thread continue; } diff --git a/Modules/_remote_debugging/object_reading.c b/Modules/_remote_debugging/object_reading.c index 2f465ca0cac41d..447b7fd5926064 100644 --- a/Modules/_remote_debugging/object_reading.c +++ b/Modules/_remote_debugging/object_reading.c @@ -194,9 +194,21 @@ read_py_long( return 0; } - // If the long object has inline digits, use them directly + // Validate size: reject garbage (negative or unreasonably large) + if (size < 0 || size > MAX_LONG_DIGITS) { + set_exception_cause(unwinder, PyExc_RuntimeError, + "Invalid PyLong size (corrupted remote memory)"); + return -1; + } + + // Calculate how many digits fit inline in our local buffer + Py_ssize_t ob_digit_offset = unwinder->debug_offsets.long_object.ob_digit; + Py_ssize_t inline_digits_space = SIZEOF_LONG_OBJ - ob_digit_offset; + Py_ssize_t max_inline_digits = inline_digits_space / (Py_ssize_t)sizeof(digit); + + // If the long object has inline digits that fit in our buffer, use them directly digit *digits; - if (size <= _PY_NSMALLNEGINTS + _PY_NSMALLPOSINTS) { + if (size <= max_inline_digits && size <= _PY_NSMALLNEGINTS + _PY_NSMALLPOSINTS) { // For small integers, digits are inline in the long_value.ob_digit array digits = (digit *)PyMem_RawMalloc(size * sizeof(digit)); if (!digits) { @@ -204,7 +216,7 @@ read_py_long( set_exception_cause(unwinder, PyExc_MemoryError, "Failed to allocate digits for small PyLong"); return -1; } - memcpy(digits, long_obj + unwinder->debug_offsets.long_object.ob_digit, size * sizeof(digit)); + memcpy(digits, long_obj + ob_digit_offset, size * sizeof(digit)); } else { // For larger integers, we need to read the digits separately digits = (digit *)PyMem_RawMalloc(size * sizeof(digit)); From b3f2d80569185be470e30ddb158f711143187dbf Mon Sep 17 00:00:00 2001 From: Hai Zhu <35182391+cocolato@users.noreply.github.com> Date: Sat, 27 Dec 2025 00:12:28 +0800 Subject: [PATCH 580/638] gh-134584: Eliminate redundant refcounting from `_COMPARE_OP_X` (GH-143186) --- Include/internal/pycore_opcode_metadata.h | 4 +- Include/internal/pycore_uop_ids.h | 1267 ++++++++++----------- Include/internal/pycore_uop_metadata.h | 30 +- Lib/test/test_capi/test_opt.py | 32 + Python/bytecodes.c | 16 +- Python/executor_cases.c.h | 78 +- Python/generated_cases.c.h | 36 +- Python/optimizer_bytecodes.c | 10 +- Python/optimizer_cases.c.h | 90 +- 9 files changed, 775 insertions(+), 788 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 8920dd42d7384a..7139b5a9e54483 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1381,9 +1381,9 @@ _PyOpcode_macro_expansion[256] = { [CHECK_EG_MATCH] = { .nuops = 1, .uops = { { _CHECK_EG_MATCH, OPARG_SIMPLE, 0 } } }, [CHECK_EXC_MATCH] = { .nuops = 1, .uops = { { _CHECK_EXC_MATCH, OPARG_SIMPLE, 0 } } }, [COMPARE_OP] = { .nuops = 1, .uops = { { _COMPARE_OP, OPARG_SIMPLE, 0 } } }, - [COMPARE_OP_FLOAT] = { .nuops = 3, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _COMPARE_OP_FLOAT, OPARG_SIMPLE, 1 } } }, + [COMPARE_OP_FLOAT] = { .nuops = 5, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _COMPARE_OP_FLOAT, OPARG_SIMPLE, 1 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 1 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 1 } } }, [COMPARE_OP_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _COMPARE_OP_INT, OPARG_SIMPLE, 1 }, { _POP_TOP_INT, OPARG_SIMPLE, 1 }, { _POP_TOP_INT, OPARG_SIMPLE, 1 } } }, - [COMPARE_OP_STR] = { .nuops = 3, .uops = { { _GUARD_TOS_UNICODE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_UNICODE, OPARG_SIMPLE, 0 }, { _COMPARE_OP_STR, OPARG_SIMPLE, 1 } } }, + [COMPARE_OP_STR] = { .nuops = 5, .uops = { { _GUARD_TOS_UNICODE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_UNICODE, OPARG_SIMPLE, 0 }, { _COMPARE_OP_STR, OPARG_SIMPLE, 1 }, { _POP_TOP_UNICODE, OPARG_SIMPLE, 1 }, { _POP_TOP_UNICODE, OPARG_SIMPLE, 1 } } }, [CONTAINS_OP] = { .nuops = 1, .uops = { { _CONTAINS_OP, OPARG_SIMPLE, 0 } } }, [CONTAINS_OP_DICT] = { .nuops = 2, .uops = { { _GUARD_TOS_DICT, OPARG_SIMPLE, 0 }, { _CONTAINS_OP_DICT, OPARG_SIMPLE, 1 } } }, [CONTAINS_OP_SET] = { .nuops = 2, .uops = { { _GUARD_TOS_ANY_SET, OPARG_SIMPLE, 0 }, { _CONTAINS_OP_SET, OPARG_SIMPLE, 1 } } }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 42fb9464b6a13d..9b3c5fbcf8d4e0 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -477,640 +477,639 @@ extern "C" { #define _COLD_DYNAMIC_EXIT_r00 670 #define _COLD_EXIT_r00 671 #define _COMPARE_OP_r21 672 -#define _COMPARE_OP_FLOAT_r01 673 -#define _COMPARE_OP_FLOAT_r11 674 -#define _COMPARE_OP_FLOAT_r21 675 -#define _COMPARE_OP_FLOAT_r32 676 -#define _COMPARE_OP_INT_r23 677 -#define _COMPARE_OP_STR_r21 678 -#define _CONTAINS_OP_r21 679 -#define _CONTAINS_OP_DICT_r21 680 -#define _CONTAINS_OP_SET_r21 681 -#define _CONVERT_VALUE_r11 682 -#define _COPY_r01 683 -#define _COPY_1_r02 684 -#define _COPY_1_r12 685 -#define _COPY_1_r23 686 -#define _COPY_2_r03 687 -#define _COPY_2_r13 688 -#define _COPY_2_r23 689 -#define _COPY_3_r03 690 -#define _COPY_3_r13 691 -#define _COPY_3_r23 692 -#define _COPY_3_r33 693 -#define _COPY_FREE_VARS_r00 694 -#define _COPY_FREE_VARS_r11 695 -#define _COPY_FREE_VARS_r22 696 -#define _COPY_FREE_VARS_r33 697 -#define _CREATE_INIT_FRAME_r01 698 -#define _DELETE_ATTR_r10 699 -#define _DELETE_DEREF_r00 700 -#define _DELETE_FAST_r00 701 -#define _DELETE_GLOBAL_r00 702 -#define _DELETE_NAME_r00 703 -#define _DELETE_SUBSCR_r20 704 -#define _DEOPT_r00 705 -#define _DEOPT_r10 706 -#define _DEOPT_r20 707 -#define _DEOPT_r30 708 -#define _DICT_MERGE_r10 709 -#define _DICT_UPDATE_r10 710 -#define _DO_CALL_r01 711 -#define _DO_CALL_FUNCTION_EX_r31 712 -#define _DO_CALL_KW_r11 713 -#define _DYNAMIC_EXIT_r00 714 -#define _DYNAMIC_EXIT_r10 715 -#define _DYNAMIC_EXIT_r20 716 -#define _DYNAMIC_EXIT_r30 717 -#define _END_FOR_r10 718 -#define _END_SEND_r21 719 -#define _ERROR_POP_N_r00 720 -#define _EXIT_INIT_CHECK_r10 721 -#define _EXIT_TRACE_r00 722 -#define _EXIT_TRACE_r10 723 -#define _EXIT_TRACE_r20 724 -#define _EXIT_TRACE_r30 725 -#define _EXPAND_METHOD_r00 726 -#define _EXPAND_METHOD_KW_r11 727 -#define _FATAL_ERROR_r00 728 -#define _FATAL_ERROR_r11 729 -#define _FATAL_ERROR_r22 730 -#define _FATAL_ERROR_r33 731 -#define _FORMAT_SIMPLE_r11 732 -#define _FORMAT_WITH_SPEC_r21 733 -#define _FOR_ITER_r23 734 -#define _FOR_ITER_GEN_FRAME_r03 735 -#define _FOR_ITER_GEN_FRAME_r13 736 -#define _FOR_ITER_GEN_FRAME_r23 737 -#define _FOR_ITER_TIER_TWO_r23 738 -#define _GET_AITER_r11 739 -#define _GET_ANEXT_r12 740 -#define _GET_AWAITABLE_r11 741 -#define _GET_ITER_r12 742 -#define _GET_LEN_r12 743 -#define _GET_YIELD_FROM_ITER_r11 744 -#define _GUARD_BINARY_OP_EXTEND_r22 745 -#define _GUARD_CALLABLE_ISINSTANCE_r03 746 -#define _GUARD_CALLABLE_ISINSTANCE_r13 747 -#define _GUARD_CALLABLE_ISINSTANCE_r23 748 -#define _GUARD_CALLABLE_ISINSTANCE_r33 749 -#define _GUARD_CALLABLE_LEN_r03 750 -#define _GUARD_CALLABLE_LEN_r13 751 -#define _GUARD_CALLABLE_LEN_r23 752 -#define _GUARD_CALLABLE_LEN_r33 753 -#define _GUARD_CALLABLE_LIST_APPEND_r03 754 -#define _GUARD_CALLABLE_LIST_APPEND_r13 755 -#define _GUARD_CALLABLE_LIST_APPEND_r23 756 -#define _GUARD_CALLABLE_LIST_APPEND_r33 757 -#define _GUARD_CALLABLE_STR_1_r03 758 -#define _GUARD_CALLABLE_STR_1_r13 759 -#define _GUARD_CALLABLE_STR_1_r23 760 -#define _GUARD_CALLABLE_STR_1_r33 761 -#define _GUARD_CALLABLE_TUPLE_1_r03 762 -#define _GUARD_CALLABLE_TUPLE_1_r13 763 -#define _GUARD_CALLABLE_TUPLE_1_r23 764 -#define _GUARD_CALLABLE_TUPLE_1_r33 765 -#define _GUARD_CALLABLE_TYPE_1_r03 766 -#define _GUARD_CALLABLE_TYPE_1_r13 767 -#define _GUARD_CALLABLE_TYPE_1_r23 768 -#define _GUARD_CALLABLE_TYPE_1_r33 769 -#define _GUARD_DORV_NO_DICT_r01 770 -#define _GUARD_DORV_NO_DICT_r11 771 -#define _GUARD_DORV_NO_DICT_r22 772 -#define _GUARD_DORV_NO_DICT_r33 773 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r01 774 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11 775 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22 776 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33 777 -#define _GUARD_GLOBALS_VERSION_r00 778 -#define _GUARD_GLOBALS_VERSION_r11 779 -#define _GUARD_GLOBALS_VERSION_r22 780 -#define _GUARD_GLOBALS_VERSION_r33 781 -#define _GUARD_IP_RETURN_GENERATOR_r00 782 -#define _GUARD_IP_RETURN_GENERATOR_r11 783 -#define _GUARD_IP_RETURN_GENERATOR_r22 784 -#define _GUARD_IP_RETURN_GENERATOR_r33 785 -#define _GUARD_IP_RETURN_VALUE_r00 786 -#define _GUARD_IP_RETURN_VALUE_r11 787 -#define _GUARD_IP_RETURN_VALUE_r22 788 -#define _GUARD_IP_RETURN_VALUE_r33 789 -#define _GUARD_IP_YIELD_VALUE_r00 790 -#define _GUARD_IP_YIELD_VALUE_r11 791 -#define _GUARD_IP_YIELD_VALUE_r22 792 -#define _GUARD_IP_YIELD_VALUE_r33 793 -#define _GUARD_IP__PUSH_FRAME_r00 794 -#define _GUARD_IP__PUSH_FRAME_r11 795 -#define _GUARD_IP__PUSH_FRAME_r22 796 -#define _GUARD_IP__PUSH_FRAME_r33 797 -#define _GUARD_IS_FALSE_POP_r00 798 -#define _GUARD_IS_FALSE_POP_r10 799 -#define _GUARD_IS_FALSE_POP_r21 800 -#define _GUARD_IS_FALSE_POP_r32 801 -#define _GUARD_IS_NONE_POP_r00 802 -#define _GUARD_IS_NONE_POP_r10 803 -#define _GUARD_IS_NONE_POP_r21 804 -#define _GUARD_IS_NONE_POP_r32 805 -#define _GUARD_IS_NOT_NONE_POP_r10 806 -#define _GUARD_IS_TRUE_POP_r00 807 -#define _GUARD_IS_TRUE_POP_r10 808 -#define _GUARD_IS_TRUE_POP_r21 809 -#define _GUARD_IS_TRUE_POP_r32 810 -#define _GUARD_KEYS_VERSION_r01 811 -#define _GUARD_KEYS_VERSION_r11 812 -#define _GUARD_KEYS_VERSION_r22 813 -#define _GUARD_KEYS_VERSION_r33 814 -#define _GUARD_NOS_DICT_r02 815 -#define _GUARD_NOS_DICT_r12 816 -#define _GUARD_NOS_DICT_r22 817 -#define _GUARD_NOS_DICT_r33 818 -#define _GUARD_NOS_FLOAT_r02 819 -#define _GUARD_NOS_FLOAT_r12 820 -#define _GUARD_NOS_FLOAT_r22 821 -#define _GUARD_NOS_FLOAT_r33 822 -#define _GUARD_NOS_INT_r02 823 -#define _GUARD_NOS_INT_r12 824 -#define _GUARD_NOS_INT_r22 825 -#define _GUARD_NOS_INT_r33 826 -#define _GUARD_NOS_LIST_r02 827 -#define _GUARD_NOS_LIST_r12 828 -#define _GUARD_NOS_LIST_r22 829 -#define _GUARD_NOS_LIST_r33 830 -#define _GUARD_NOS_NOT_NULL_r02 831 -#define _GUARD_NOS_NOT_NULL_r12 832 -#define _GUARD_NOS_NOT_NULL_r22 833 -#define _GUARD_NOS_NOT_NULL_r33 834 -#define _GUARD_NOS_NULL_r02 835 -#define _GUARD_NOS_NULL_r12 836 -#define _GUARD_NOS_NULL_r22 837 -#define _GUARD_NOS_NULL_r33 838 -#define _GUARD_NOS_OVERFLOWED_r02 839 -#define _GUARD_NOS_OVERFLOWED_r12 840 -#define _GUARD_NOS_OVERFLOWED_r22 841 -#define _GUARD_NOS_OVERFLOWED_r33 842 -#define _GUARD_NOS_TUPLE_r02 843 -#define _GUARD_NOS_TUPLE_r12 844 -#define _GUARD_NOS_TUPLE_r22 845 -#define _GUARD_NOS_TUPLE_r33 846 -#define _GUARD_NOS_UNICODE_r02 847 -#define _GUARD_NOS_UNICODE_r12 848 -#define _GUARD_NOS_UNICODE_r22 849 -#define _GUARD_NOS_UNICODE_r33 850 -#define _GUARD_NOT_EXHAUSTED_LIST_r02 851 -#define _GUARD_NOT_EXHAUSTED_LIST_r12 852 -#define _GUARD_NOT_EXHAUSTED_LIST_r22 853 -#define _GUARD_NOT_EXHAUSTED_LIST_r33 854 -#define _GUARD_NOT_EXHAUSTED_RANGE_r02 855 -#define _GUARD_NOT_EXHAUSTED_RANGE_r12 856 -#define _GUARD_NOT_EXHAUSTED_RANGE_r22 857 -#define _GUARD_NOT_EXHAUSTED_RANGE_r33 858 -#define _GUARD_NOT_EXHAUSTED_TUPLE_r02 859 -#define _GUARD_NOT_EXHAUSTED_TUPLE_r12 860 -#define _GUARD_NOT_EXHAUSTED_TUPLE_r22 861 -#define _GUARD_NOT_EXHAUSTED_TUPLE_r33 862 -#define _GUARD_THIRD_NULL_r03 863 -#define _GUARD_THIRD_NULL_r13 864 -#define _GUARD_THIRD_NULL_r23 865 -#define _GUARD_THIRD_NULL_r33 866 -#define _GUARD_TOS_ANY_SET_r01 867 -#define _GUARD_TOS_ANY_SET_r11 868 -#define _GUARD_TOS_ANY_SET_r22 869 -#define _GUARD_TOS_ANY_SET_r33 870 -#define _GUARD_TOS_DICT_r01 871 -#define _GUARD_TOS_DICT_r11 872 -#define _GUARD_TOS_DICT_r22 873 -#define _GUARD_TOS_DICT_r33 874 -#define _GUARD_TOS_FLOAT_r01 875 -#define _GUARD_TOS_FLOAT_r11 876 -#define _GUARD_TOS_FLOAT_r22 877 -#define _GUARD_TOS_FLOAT_r33 878 -#define _GUARD_TOS_INT_r01 879 -#define _GUARD_TOS_INT_r11 880 -#define _GUARD_TOS_INT_r22 881 -#define _GUARD_TOS_INT_r33 882 -#define _GUARD_TOS_LIST_r01 883 -#define _GUARD_TOS_LIST_r11 884 -#define _GUARD_TOS_LIST_r22 885 -#define _GUARD_TOS_LIST_r33 886 -#define _GUARD_TOS_OVERFLOWED_r01 887 -#define _GUARD_TOS_OVERFLOWED_r11 888 -#define _GUARD_TOS_OVERFLOWED_r22 889 -#define _GUARD_TOS_OVERFLOWED_r33 890 -#define _GUARD_TOS_SLICE_r01 891 -#define _GUARD_TOS_SLICE_r11 892 -#define _GUARD_TOS_SLICE_r22 893 -#define _GUARD_TOS_SLICE_r33 894 -#define _GUARD_TOS_TUPLE_r01 895 -#define _GUARD_TOS_TUPLE_r11 896 -#define _GUARD_TOS_TUPLE_r22 897 -#define _GUARD_TOS_TUPLE_r33 898 -#define _GUARD_TOS_UNICODE_r01 899 -#define _GUARD_TOS_UNICODE_r11 900 -#define _GUARD_TOS_UNICODE_r22 901 -#define _GUARD_TOS_UNICODE_r33 902 -#define _GUARD_TYPE_VERSION_r01 903 -#define _GUARD_TYPE_VERSION_r11 904 -#define _GUARD_TYPE_VERSION_r22 905 -#define _GUARD_TYPE_VERSION_r33 906 -#define _GUARD_TYPE_VERSION_AND_LOCK_r01 907 -#define _GUARD_TYPE_VERSION_AND_LOCK_r11 908 -#define _GUARD_TYPE_VERSION_AND_LOCK_r22 909 -#define _GUARD_TYPE_VERSION_AND_LOCK_r33 910 -#define _HANDLE_PENDING_AND_DEOPT_r00 911 -#define _HANDLE_PENDING_AND_DEOPT_r10 912 -#define _HANDLE_PENDING_AND_DEOPT_r20 913 -#define _HANDLE_PENDING_AND_DEOPT_r30 914 -#define _IMPORT_FROM_r12 915 -#define _IMPORT_NAME_r21 916 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00 917 -#define _INIT_CALL_PY_EXACT_ARGS_r01 918 -#define _INIT_CALL_PY_EXACT_ARGS_0_r01 919 -#define _INIT_CALL_PY_EXACT_ARGS_1_r01 920 -#define _INIT_CALL_PY_EXACT_ARGS_2_r01 921 -#define _INIT_CALL_PY_EXACT_ARGS_3_r01 922 -#define _INIT_CALL_PY_EXACT_ARGS_4_r01 923 -#define _INSERT_NULL_r10 924 -#define _INSTRUMENTED_FOR_ITER_r23 925 -#define _INSTRUMENTED_INSTRUCTION_r00 926 -#define _INSTRUMENTED_JUMP_FORWARD_r00 927 -#define _INSTRUMENTED_JUMP_FORWARD_r11 928 -#define _INSTRUMENTED_JUMP_FORWARD_r22 929 -#define _INSTRUMENTED_JUMP_FORWARD_r33 930 -#define _INSTRUMENTED_LINE_r00 931 -#define _INSTRUMENTED_NOT_TAKEN_r00 932 -#define _INSTRUMENTED_NOT_TAKEN_r11 933 -#define _INSTRUMENTED_NOT_TAKEN_r22 934 -#define _INSTRUMENTED_NOT_TAKEN_r33 935 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r00 936 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r10 937 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r21 938 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r32 939 -#define _INSTRUMENTED_POP_JUMP_IF_NONE_r10 940 -#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE_r10 941 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r00 942 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r10 943 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r21 944 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r32 945 -#define _IS_NONE_r11 946 -#define _IS_OP_r21 947 -#define _ITER_CHECK_LIST_r02 948 -#define _ITER_CHECK_LIST_r12 949 -#define _ITER_CHECK_LIST_r22 950 -#define _ITER_CHECK_LIST_r33 951 -#define _ITER_CHECK_RANGE_r02 952 -#define _ITER_CHECK_RANGE_r12 953 -#define _ITER_CHECK_RANGE_r22 954 -#define _ITER_CHECK_RANGE_r33 955 -#define _ITER_CHECK_TUPLE_r02 956 -#define _ITER_CHECK_TUPLE_r12 957 -#define _ITER_CHECK_TUPLE_r22 958 -#define _ITER_CHECK_TUPLE_r33 959 -#define _ITER_JUMP_LIST_r02 960 -#define _ITER_JUMP_LIST_r12 961 -#define _ITER_JUMP_LIST_r22 962 -#define _ITER_JUMP_LIST_r33 963 -#define _ITER_JUMP_RANGE_r02 964 -#define _ITER_JUMP_RANGE_r12 965 -#define _ITER_JUMP_RANGE_r22 966 -#define _ITER_JUMP_RANGE_r33 967 -#define _ITER_JUMP_TUPLE_r02 968 -#define _ITER_JUMP_TUPLE_r12 969 -#define _ITER_JUMP_TUPLE_r22 970 -#define _ITER_JUMP_TUPLE_r33 971 -#define _ITER_NEXT_LIST_r23 972 -#define _ITER_NEXT_LIST_TIER_TWO_r23 973 -#define _ITER_NEXT_RANGE_r03 974 -#define _ITER_NEXT_RANGE_r13 975 -#define _ITER_NEXT_RANGE_r23 976 -#define _ITER_NEXT_TUPLE_r03 977 -#define _ITER_NEXT_TUPLE_r13 978 -#define _ITER_NEXT_TUPLE_r23 979 -#define _JUMP_BACKWARD_NO_INTERRUPT_r00 980 -#define _JUMP_BACKWARD_NO_INTERRUPT_r11 981 -#define _JUMP_BACKWARD_NO_INTERRUPT_r22 982 -#define _JUMP_BACKWARD_NO_INTERRUPT_r33 983 -#define _JUMP_TO_TOP_r00 984 -#define _LIST_APPEND_r10 985 -#define _LIST_EXTEND_r10 986 -#define _LOAD_ATTR_r10 987 -#define _LOAD_ATTR_CLASS_r11 988 -#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_r11 989 -#define _LOAD_ATTR_INSTANCE_VALUE_r02 990 -#define _LOAD_ATTR_INSTANCE_VALUE_r12 991 -#define _LOAD_ATTR_INSTANCE_VALUE_r23 992 -#define _LOAD_ATTR_METHOD_LAZY_DICT_r02 993 -#define _LOAD_ATTR_METHOD_LAZY_DICT_r12 994 -#define _LOAD_ATTR_METHOD_LAZY_DICT_r23 995 -#define _LOAD_ATTR_METHOD_NO_DICT_r02 996 -#define _LOAD_ATTR_METHOD_NO_DICT_r12 997 -#define _LOAD_ATTR_METHOD_NO_DICT_r23 998 -#define _LOAD_ATTR_METHOD_WITH_VALUES_r02 999 -#define _LOAD_ATTR_METHOD_WITH_VALUES_r12 1000 -#define _LOAD_ATTR_METHOD_WITH_VALUES_r23 1001 -#define _LOAD_ATTR_MODULE_r11 1002 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11 1003 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11 1004 -#define _LOAD_ATTR_PROPERTY_FRAME_r11 1005 -#define _LOAD_ATTR_SLOT_r11 1006 -#define _LOAD_ATTR_WITH_HINT_r12 1007 -#define _LOAD_BUILD_CLASS_r01 1008 -#define _LOAD_BYTECODE_r00 1009 -#define _LOAD_COMMON_CONSTANT_r01 1010 -#define _LOAD_COMMON_CONSTANT_r12 1011 -#define _LOAD_COMMON_CONSTANT_r23 1012 -#define _LOAD_CONST_r01 1013 -#define _LOAD_CONST_r12 1014 -#define _LOAD_CONST_r23 1015 -#define _LOAD_CONST_INLINE_r01 1016 -#define _LOAD_CONST_INLINE_r12 1017 -#define _LOAD_CONST_INLINE_r23 1018 -#define _LOAD_CONST_INLINE_BORROW_r01 1019 -#define _LOAD_CONST_INLINE_BORROW_r12 1020 -#define _LOAD_CONST_INLINE_BORROW_r23 1021 -#define _LOAD_CONST_UNDER_INLINE_r02 1022 -#define _LOAD_CONST_UNDER_INLINE_r12 1023 -#define _LOAD_CONST_UNDER_INLINE_r23 1024 -#define _LOAD_CONST_UNDER_INLINE_BORROW_r02 1025 -#define _LOAD_CONST_UNDER_INLINE_BORROW_r12 1026 -#define _LOAD_CONST_UNDER_INLINE_BORROW_r23 1027 -#define _LOAD_DEREF_r01 1028 -#define _LOAD_FAST_r01 1029 -#define _LOAD_FAST_r12 1030 -#define _LOAD_FAST_r23 1031 -#define _LOAD_FAST_0_r01 1032 -#define _LOAD_FAST_0_r12 1033 -#define _LOAD_FAST_0_r23 1034 -#define _LOAD_FAST_1_r01 1035 -#define _LOAD_FAST_1_r12 1036 -#define _LOAD_FAST_1_r23 1037 -#define _LOAD_FAST_2_r01 1038 -#define _LOAD_FAST_2_r12 1039 -#define _LOAD_FAST_2_r23 1040 -#define _LOAD_FAST_3_r01 1041 -#define _LOAD_FAST_3_r12 1042 -#define _LOAD_FAST_3_r23 1043 -#define _LOAD_FAST_4_r01 1044 -#define _LOAD_FAST_4_r12 1045 -#define _LOAD_FAST_4_r23 1046 -#define _LOAD_FAST_5_r01 1047 -#define _LOAD_FAST_5_r12 1048 -#define _LOAD_FAST_5_r23 1049 -#define _LOAD_FAST_6_r01 1050 -#define _LOAD_FAST_6_r12 1051 -#define _LOAD_FAST_6_r23 1052 -#define _LOAD_FAST_7_r01 1053 -#define _LOAD_FAST_7_r12 1054 -#define _LOAD_FAST_7_r23 1055 -#define _LOAD_FAST_AND_CLEAR_r01 1056 -#define _LOAD_FAST_AND_CLEAR_r12 1057 -#define _LOAD_FAST_AND_CLEAR_r23 1058 -#define _LOAD_FAST_BORROW_r01 1059 -#define _LOAD_FAST_BORROW_r12 1060 -#define _LOAD_FAST_BORROW_r23 1061 -#define _LOAD_FAST_BORROW_0_r01 1062 -#define _LOAD_FAST_BORROW_0_r12 1063 -#define _LOAD_FAST_BORROW_0_r23 1064 -#define _LOAD_FAST_BORROW_1_r01 1065 -#define _LOAD_FAST_BORROW_1_r12 1066 -#define _LOAD_FAST_BORROW_1_r23 1067 -#define _LOAD_FAST_BORROW_2_r01 1068 -#define _LOAD_FAST_BORROW_2_r12 1069 -#define _LOAD_FAST_BORROW_2_r23 1070 -#define _LOAD_FAST_BORROW_3_r01 1071 -#define _LOAD_FAST_BORROW_3_r12 1072 -#define _LOAD_FAST_BORROW_3_r23 1073 -#define _LOAD_FAST_BORROW_4_r01 1074 -#define _LOAD_FAST_BORROW_4_r12 1075 -#define _LOAD_FAST_BORROW_4_r23 1076 -#define _LOAD_FAST_BORROW_5_r01 1077 -#define _LOAD_FAST_BORROW_5_r12 1078 -#define _LOAD_FAST_BORROW_5_r23 1079 -#define _LOAD_FAST_BORROW_6_r01 1080 -#define _LOAD_FAST_BORROW_6_r12 1081 -#define _LOAD_FAST_BORROW_6_r23 1082 -#define _LOAD_FAST_BORROW_7_r01 1083 -#define _LOAD_FAST_BORROW_7_r12 1084 -#define _LOAD_FAST_BORROW_7_r23 1085 -#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r02 1086 -#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r13 1087 -#define _LOAD_FAST_CHECK_r01 1088 -#define _LOAD_FAST_CHECK_r12 1089 -#define _LOAD_FAST_CHECK_r23 1090 -#define _LOAD_FAST_LOAD_FAST_r02 1091 -#define _LOAD_FAST_LOAD_FAST_r13 1092 -#define _LOAD_FROM_DICT_OR_DEREF_r11 1093 -#define _LOAD_FROM_DICT_OR_GLOBALS_r11 1094 -#define _LOAD_GLOBAL_r00 1095 -#define _LOAD_GLOBAL_BUILTINS_r01 1096 -#define _LOAD_GLOBAL_MODULE_r01 1097 -#define _LOAD_LOCALS_r01 1098 -#define _LOAD_LOCALS_r12 1099 -#define _LOAD_LOCALS_r23 1100 -#define _LOAD_NAME_r01 1101 -#define _LOAD_SMALL_INT_r01 1102 -#define _LOAD_SMALL_INT_r12 1103 -#define _LOAD_SMALL_INT_r23 1104 -#define _LOAD_SMALL_INT_0_r01 1105 -#define _LOAD_SMALL_INT_0_r12 1106 -#define _LOAD_SMALL_INT_0_r23 1107 -#define _LOAD_SMALL_INT_1_r01 1108 -#define _LOAD_SMALL_INT_1_r12 1109 -#define _LOAD_SMALL_INT_1_r23 1110 -#define _LOAD_SMALL_INT_2_r01 1111 -#define _LOAD_SMALL_INT_2_r12 1112 -#define _LOAD_SMALL_INT_2_r23 1113 -#define _LOAD_SMALL_INT_3_r01 1114 -#define _LOAD_SMALL_INT_3_r12 1115 -#define _LOAD_SMALL_INT_3_r23 1116 -#define _LOAD_SPECIAL_r00 1117 -#define _LOAD_SUPER_ATTR_ATTR_r31 1118 -#define _LOAD_SUPER_ATTR_METHOD_r32 1119 -#define _MAKE_CALLARGS_A_TUPLE_r33 1120 -#define _MAKE_CELL_r00 1121 -#define _MAKE_FUNCTION_r11 1122 -#define _MAKE_WARM_r00 1123 -#define _MAKE_WARM_r11 1124 -#define _MAKE_WARM_r22 1125 -#define _MAKE_WARM_r33 1126 -#define _MAP_ADD_r20 1127 -#define _MATCH_CLASS_r31 1128 -#define _MATCH_KEYS_r23 1129 -#define _MATCH_MAPPING_r02 1130 -#define _MATCH_MAPPING_r12 1131 -#define _MATCH_MAPPING_r23 1132 -#define _MATCH_SEQUENCE_r02 1133 -#define _MATCH_SEQUENCE_r12 1134 -#define _MATCH_SEQUENCE_r23 1135 -#define _MAYBE_EXPAND_METHOD_r00 1136 -#define _MAYBE_EXPAND_METHOD_KW_r11 1137 -#define _MONITOR_CALL_r00 1138 -#define _MONITOR_CALL_KW_r11 1139 -#define _MONITOR_JUMP_BACKWARD_r00 1140 -#define _MONITOR_JUMP_BACKWARD_r11 1141 -#define _MONITOR_JUMP_BACKWARD_r22 1142 -#define _MONITOR_JUMP_BACKWARD_r33 1143 -#define _MONITOR_RESUME_r00 1144 -#define _NOP_r00 1145 -#define _NOP_r11 1146 -#define _NOP_r22 1147 -#define _NOP_r33 1148 -#define _POP_CALL_r20 1149 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW_r21 1150 -#define _POP_CALL_ONE_r30 1151 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW_r31 1152 -#define _POP_CALL_TWO_r30 1153 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW_r31 1154 -#define _POP_EXCEPT_r10 1155 -#define _POP_ITER_r20 1156 -#define _POP_JUMP_IF_FALSE_r00 1157 -#define _POP_JUMP_IF_FALSE_r10 1158 -#define _POP_JUMP_IF_FALSE_r21 1159 -#define _POP_JUMP_IF_FALSE_r32 1160 -#define _POP_JUMP_IF_TRUE_r00 1161 -#define _POP_JUMP_IF_TRUE_r10 1162 -#define _POP_JUMP_IF_TRUE_r21 1163 -#define _POP_JUMP_IF_TRUE_r32 1164 -#define _POP_TOP_r10 1165 -#define _POP_TOP_FLOAT_r00 1166 -#define _POP_TOP_FLOAT_r10 1167 -#define _POP_TOP_FLOAT_r21 1168 -#define _POP_TOP_FLOAT_r32 1169 -#define _POP_TOP_INT_r00 1170 -#define _POP_TOP_INT_r10 1171 -#define _POP_TOP_INT_r21 1172 -#define _POP_TOP_INT_r32 1173 -#define _POP_TOP_LOAD_CONST_INLINE_r11 1174 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW_r11 1175 -#define _POP_TOP_NOP_r00 1176 -#define _POP_TOP_NOP_r10 1177 -#define _POP_TOP_NOP_r21 1178 -#define _POP_TOP_NOP_r32 1179 -#define _POP_TOP_UNICODE_r00 1180 -#define _POP_TOP_UNICODE_r10 1181 -#define _POP_TOP_UNICODE_r21 1182 -#define _POP_TOP_UNICODE_r32 1183 -#define _POP_TWO_r20 1184 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW_r21 1185 -#define _PUSH_EXC_INFO_r02 1186 -#define _PUSH_EXC_INFO_r12 1187 -#define _PUSH_EXC_INFO_r23 1188 -#define _PUSH_FRAME_r10 1189 -#define _PUSH_NULL_r01 1190 -#define _PUSH_NULL_r12 1191 -#define _PUSH_NULL_r23 1192 -#define _PUSH_NULL_CONDITIONAL_r00 1193 -#define _PY_FRAME_GENERAL_r01 1194 -#define _PY_FRAME_KW_r11 1195 -#define _QUICKEN_RESUME_r00 1196 -#define _QUICKEN_RESUME_r11 1197 -#define _QUICKEN_RESUME_r22 1198 -#define _QUICKEN_RESUME_r33 1199 -#define _REPLACE_WITH_TRUE_r11 1200 -#define _RESUME_CHECK_r00 1201 -#define _RESUME_CHECK_r11 1202 -#define _RESUME_CHECK_r22 1203 -#define _RESUME_CHECK_r33 1204 -#define _RETURN_GENERATOR_r01 1205 -#define _RETURN_VALUE_r11 1206 -#define _SAVE_RETURN_OFFSET_r00 1207 -#define _SAVE_RETURN_OFFSET_r11 1208 -#define _SAVE_RETURN_OFFSET_r22 1209 -#define _SAVE_RETURN_OFFSET_r33 1210 -#define _SEND_r22 1211 -#define _SEND_GEN_FRAME_r22 1212 -#define _SETUP_ANNOTATIONS_r00 1213 -#define _SET_ADD_r10 1214 -#define _SET_FUNCTION_ATTRIBUTE_r01 1215 -#define _SET_FUNCTION_ATTRIBUTE_r11 1216 -#define _SET_FUNCTION_ATTRIBUTE_r21 1217 -#define _SET_FUNCTION_ATTRIBUTE_r32 1218 -#define _SET_IP_r00 1219 -#define _SET_IP_r11 1220 -#define _SET_IP_r22 1221 -#define _SET_IP_r33 1222 -#define _SET_UPDATE_r10 1223 -#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r02 1224 -#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r12 1225 -#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r22 1226 -#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r32 1227 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03 1228 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13 1229 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23 1230 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33 1231 -#define _SPILL_OR_RELOAD_r01 1232 -#define _SPILL_OR_RELOAD_r02 1233 -#define _SPILL_OR_RELOAD_r03 1234 -#define _SPILL_OR_RELOAD_r10 1235 -#define _SPILL_OR_RELOAD_r12 1236 -#define _SPILL_OR_RELOAD_r13 1237 -#define _SPILL_OR_RELOAD_r20 1238 -#define _SPILL_OR_RELOAD_r21 1239 -#define _SPILL_OR_RELOAD_r23 1240 -#define _SPILL_OR_RELOAD_r30 1241 -#define _SPILL_OR_RELOAD_r31 1242 -#define _SPILL_OR_RELOAD_r32 1243 -#define _START_EXECUTOR_r00 1244 -#define _STORE_ATTR_r20 1245 -#define _STORE_ATTR_INSTANCE_VALUE_r21 1246 -#define _STORE_ATTR_SLOT_r21 1247 -#define _STORE_ATTR_WITH_HINT_r21 1248 -#define _STORE_DEREF_r10 1249 -#define _STORE_FAST_r10 1250 -#define _STORE_FAST_0_r10 1251 -#define _STORE_FAST_1_r10 1252 -#define _STORE_FAST_2_r10 1253 -#define _STORE_FAST_3_r10 1254 -#define _STORE_FAST_4_r10 1255 -#define _STORE_FAST_5_r10 1256 -#define _STORE_FAST_6_r10 1257 -#define _STORE_FAST_7_r10 1258 -#define _STORE_FAST_LOAD_FAST_r11 1259 -#define _STORE_FAST_STORE_FAST_r20 1260 -#define _STORE_GLOBAL_r10 1261 -#define _STORE_NAME_r10 1262 -#define _STORE_SLICE_r30 1263 -#define _STORE_SUBSCR_r30 1264 -#define _STORE_SUBSCR_DICT_r31 1265 -#define _STORE_SUBSCR_LIST_INT_r32 1266 -#define _SWAP_r11 1267 -#define _SWAP_2_r02 1268 -#define _SWAP_2_r12 1269 -#define _SWAP_2_r22 1270 -#define _SWAP_2_r33 1271 -#define _SWAP_3_r03 1272 -#define _SWAP_3_r13 1273 -#define _SWAP_3_r23 1274 -#define _SWAP_3_r33 1275 -#define _TIER2_RESUME_CHECK_r00 1276 -#define _TIER2_RESUME_CHECK_r11 1277 -#define _TIER2_RESUME_CHECK_r22 1278 -#define _TIER2_RESUME_CHECK_r33 1279 -#define _TO_BOOL_r11 1280 -#define _TO_BOOL_BOOL_r01 1281 -#define _TO_BOOL_BOOL_r11 1282 -#define _TO_BOOL_BOOL_r22 1283 -#define _TO_BOOL_BOOL_r33 1284 -#define _TO_BOOL_INT_r11 1285 -#define _TO_BOOL_LIST_r11 1286 -#define _TO_BOOL_NONE_r01 1287 -#define _TO_BOOL_NONE_r11 1288 -#define _TO_BOOL_NONE_r22 1289 -#define _TO_BOOL_NONE_r33 1290 -#define _TO_BOOL_STR_r11 1291 -#define _TRACE_RECORD_r00 1292 -#define _UNARY_INVERT_r11 1293 -#define _UNARY_NEGATIVE_r11 1294 -#define _UNARY_NOT_r01 1295 -#define _UNARY_NOT_r11 1296 -#define _UNARY_NOT_r22 1297 -#define _UNARY_NOT_r33 1298 -#define _UNPACK_EX_r10 1299 -#define _UNPACK_SEQUENCE_r10 1300 -#define _UNPACK_SEQUENCE_LIST_r10 1301 -#define _UNPACK_SEQUENCE_TUPLE_r10 1302 -#define _UNPACK_SEQUENCE_TWO_TUPLE_r12 1303 -#define _WITH_EXCEPT_START_r33 1304 -#define _YIELD_VALUE_r11 1305 -#define MAX_UOP_REGS_ID 1305 +#define _COMPARE_OP_FLOAT_r03 673 +#define _COMPARE_OP_FLOAT_r13 674 +#define _COMPARE_OP_FLOAT_r23 675 +#define _COMPARE_OP_INT_r23 676 +#define _COMPARE_OP_STR_r23 677 +#define _CONTAINS_OP_r21 678 +#define _CONTAINS_OP_DICT_r21 679 +#define _CONTAINS_OP_SET_r21 680 +#define _CONVERT_VALUE_r11 681 +#define _COPY_r01 682 +#define _COPY_1_r02 683 +#define _COPY_1_r12 684 +#define _COPY_1_r23 685 +#define _COPY_2_r03 686 +#define _COPY_2_r13 687 +#define _COPY_2_r23 688 +#define _COPY_3_r03 689 +#define _COPY_3_r13 690 +#define _COPY_3_r23 691 +#define _COPY_3_r33 692 +#define _COPY_FREE_VARS_r00 693 +#define _COPY_FREE_VARS_r11 694 +#define _COPY_FREE_VARS_r22 695 +#define _COPY_FREE_VARS_r33 696 +#define _CREATE_INIT_FRAME_r01 697 +#define _DELETE_ATTR_r10 698 +#define _DELETE_DEREF_r00 699 +#define _DELETE_FAST_r00 700 +#define _DELETE_GLOBAL_r00 701 +#define _DELETE_NAME_r00 702 +#define _DELETE_SUBSCR_r20 703 +#define _DEOPT_r00 704 +#define _DEOPT_r10 705 +#define _DEOPT_r20 706 +#define _DEOPT_r30 707 +#define _DICT_MERGE_r10 708 +#define _DICT_UPDATE_r10 709 +#define _DO_CALL_r01 710 +#define _DO_CALL_FUNCTION_EX_r31 711 +#define _DO_CALL_KW_r11 712 +#define _DYNAMIC_EXIT_r00 713 +#define _DYNAMIC_EXIT_r10 714 +#define _DYNAMIC_EXIT_r20 715 +#define _DYNAMIC_EXIT_r30 716 +#define _END_FOR_r10 717 +#define _END_SEND_r21 718 +#define _ERROR_POP_N_r00 719 +#define _EXIT_INIT_CHECK_r10 720 +#define _EXIT_TRACE_r00 721 +#define _EXIT_TRACE_r10 722 +#define _EXIT_TRACE_r20 723 +#define _EXIT_TRACE_r30 724 +#define _EXPAND_METHOD_r00 725 +#define _EXPAND_METHOD_KW_r11 726 +#define _FATAL_ERROR_r00 727 +#define _FATAL_ERROR_r11 728 +#define _FATAL_ERROR_r22 729 +#define _FATAL_ERROR_r33 730 +#define _FORMAT_SIMPLE_r11 731 +#define _FORMAT_WITH_SPEC_r21 732 +#define _FOR_ITER_r23 733 +#define _FOR_ITER_GEN_FRAME_r03 734 +#define _FOR_ITER_GEN_FRAME_r13 735 +#define _FOR_ITER_GEN_FRAME_r23 736 +#define _FOR_ITER_TIER_TWO_r23 737 +#define _GET_AITER_r11 738 +#define _GET_ANEXT_r12 739 +#define _GET_AWAITABLE_r11 740 +#define _GET_ITER_r12 741 +#define _GET_LEN_r12 742 +#define _GET_YIELD_FROM_ITER_r11 743 +#define _GUARD_BINARY_OP_EXTEND_r22 744 +#define _GUARD_CALLABLE_ISINSTANCE_r03 745 +#define _GUARD_CALLABLE_ISINSTANCE_r13 746 +#define _GUARD_CALLABLE_ISINSTANCE_r23 747 +#define _GUARD_CALLABLE_ISINSTANCE_r33 748 +#define _GUARD_CALLABLE_LEN_r03 749 +#define _GUARD_CALLABLE_LEN_r13 750 +#define _GUARD_CALLABLE_LEN_r23 751 +#define _GUARD_CALLABLE_LEN_r33 752 +#define _GUARD_CALLABLE_LIST_APPEND_r03 753 +#define _GUARD_CALLABLE_LIST_APPEND_r13 754 +#define _GUARD_CALLABLE_LIST_APPEND_r23 755 +#define _GUARD_CALLABLE_LIST_APPEND_r33 756 +#define _GUARD_CALLABLE_STR_1_r03 757 +#define _GUARD_CALLABLE_STR_1_r13 758 +#define _GUARD_CALLABLE_STR_1_r23 759 +#define _GUARD_CALLABLE_STR_1_r33 760 +#define _GUARD_CALLABLE_TUPLE_1_r03 761 +#define _GUARD_CALLABLE_TUPLE_1_r13 762 +#define _GUARD_CALLABLE_TUPLE_1_r23 763 +#define _GUARD_CALLABLE_TUPLE_1_r33 764 +#define _GUARD_CALLABLE_TYPE_1_r03 765 +#define _GUARD_CALLABLE_TYPE_1_r13 766 +#define _GUARD_CALLABLE_TYPE_1_r23 767 +#define _GUARD_CALLABLE_TYPE_1_r33 768 +#define _GUARD_DORV_NO_DICT_r01 769 +#define _GUARD_DORV_NO_DICT_r11 770 +#define _GUARD_DORV_NO_DICT_r22 771 +#define _GUARD_DORV_NO_DICT_r33 772 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r01 773 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11 774 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22 775 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33 776 +#define _GUARD_GLOBALS_VERSION_r00 777 +#define _GUARD_GLOBALS_VERSION_r11 778 +#define _GUARD_GLOBALS_VERSION_r22 779 +#define _GUARD_GLOBALS_VERSION_r33 780 +#define _GUARD_IP_RETURN_GENERATOR_r00 781 +#define _GUARD_IP_RETURN_GENERATOR_r11 782 +#define _GUARD_IP_RETURN_GENERATOR_r22 783 +#define _GUARD_IP_RETURN_GENERATOR_r33 784 +#define _GUARD_IP_RETURN_VALUE_r00 785 +#define _GUARD_IP_RETURN_VALUE_r11 786 +#define _GUARD_IP_RETURN_VALUE_r22 787 +#define _GUARD_IP_RETURN_VALUE_r33 788 +#define _GUARD_IP_YIELD_VALUE_r00 789 +#define _GUARD_IP_YIELD_VALUE_r11 790 +#define _GUARD_IP_YIELD_VALUE_r22 791 +#define _GUARD_IP_YIELD_VALUE_r33 792 +#define _GUARD_IP__PUSH_FRAME_r00 793 +#define _GUARD_IP__PUSH_FRAME_r11 794 +#define _GUARD_IP__PUSH_FRAME_r22 795 +#define _GUARD_IP__PUSH_FRAME_r33 796 +#define _GUARD_IS_FALSE_POP_r00 797 +#define _GUARD_IS_FALSE_POP_r10 798 +#define _GUARD_IS_FALSE_POP_r21 799 +#define _GUARD_IS_FALSE_POP_r32 800 +#define _GUARD_IS_NONE_POP_r00 801 +#define _GUARD_IS_NONE_POP_r10 802 +#define _GUARD_IS_NONE_POP_r21 803 +#define _GUARD_IS_NONE_POP_r32 804 +#define _GUARD_IS_NOT_NONE_POP_r10 805 +#define _GUARD_IS_TRUE_POP_r00 806 +#define _GUARD_IS_TRUE_POP_r10 807 +#define _GUARD_IS_TRUE_POP_r21 808 +#define _GUARD_IS_TRUE_POP_r32 809 +#define _GUARD_KEYS_VERSION_r01 810 +#define _GUARD_KEYS_VERSION_r11 811 +#define _GUARD_KEYS_VERSION_r22 812 +#define _GUARD_KEYS_VERSION_r33 813 +#define _GUARD_NOS_DICT_r02 814 +#define _GUARD_NOS_DICT_r12 815 +#define _GUARD_NOS_DICT_r22 816 +#define _GUARD_NOS_DICT_r33 817 +#define _GUARD_NOS_FLOAT_r02 818 +#define _GUARD_NOS_FLOAT_r12 819 +#define _GUARD_NOS_FLOAT_r22 820 +#define _GUARD_NOS_FLOAT_r33 821 +#define _GUARD_NOS_INT_r02 822 +#define _GUARD_NOS_INT_r12 823 +#define _GUARD_NOS_INT_r22 824 +#define _GUARD_NOS_INT_r33 825 +#define _GUARD_NOS_LIST_r02 826 +#define _GUARD_NOS_LIST_r12 827 +#define _GUARD_NOS_LIST_r22 828 +#define _GUARD_NOS_LIST_r33 829 +#define _GUARD_NOS_NOT_NULL_r02 830 +#define _GUARD_NOS_NOT_NULL_r12 831 +#define _GUARD_NOS_NOT_NULL_r22 832 +#define _GUARD_NOS_NOT_NULL_r33 833 +#define _GUARD_NOS_NULL_r02 834 +#define _GUARD_NOS_NULL_r12 835 +#define _GUARD_NOS_NULL_r22 836 +#define _GUARD_NOS_NULL_r33 837 +#define _GUARD_NOS_OVERFLOWED_r02 838 +#define _GUARD_NOS_OVERFLOWED_r12 839 +#define _GUARD_NOS_OVERFLOWED_r22 840 +#define _GUARD_NOS_OVERFLOWED_r33 841 +#define _GUARD_NOS_TUPLE_r02 842 +#define _GUARD_NOS_TUPLE_r12 843 +#define _GUARD_NOS_TUPLE_r22 844 +#define _GUARD_NOS_TUPLE_r33 845 +#define _GUARD_NOS_UNICODE_r02 846 +#define _GUARD_NOS_UNICODE_r12 847 +#define _GUARD_NOS_UNICODE_r22 848 +#define _GUARD_NOS_UNICODE_r33 849 +#define _GUARD_NOT_EXHAUSTED_LIST_r02 850 +#define _GUARD_NOT_EXHAUSTED_LIST_r12 851 +#define _GUARD_NOT_EXHAUSTED_LIST_r22 852 +#define _GUARD_NOT_EXHAUSTED_LIST_r33 853 +#define _GUARD_NOT_EXHAUSTED_RANGE_r02 854 +#define _GUARD_NOT_EXHAUSTED_RANGE_r12 855 +#define _GUARD_NOT_EXHAUSTED_RANGE_r22 856 +#define _GUARD_NOT_EXHAUSTED_RANGE_r33 857 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r02 858 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r12 859 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r22 860 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r33 861 +#define _GUARD_THIRD_NULL_r03 862 +#define _GUARD_THIRD_NULL_r13 863 +#define _GUARD_THIRD_NULL_r23 864 +#define _GUARD_THIRD_NULL_r33 865 +#define _GUARD_TOS_ANY_SET_r01 866 +#define _GUARD_TOS_ANY_SET_r11 867 +#define _GUARD_TOS_ANY_SET_r22 868 +#define _GUARD_TOS_ANY_SET_r33 869 +#define _GUARD_TOS_DICT_r01 870 +#define _GUARD_TOS_DICT_r11 871 +#define _GUARD_TOS_DICT_r22 872 +#define _GUARD_TOS_DICT_r33 873 +#define _GUARD_TOS_FLOAT_r01 874 +#define _GUARD_TOS_FLOAT_r11 875 +#define _GUARD_TOS_FLOAT_r22 876 +#define _GUARD_TOS_FLOAT_r33 877 +#define _GUARD_TOS_INT_r01 878 +#define _GUARD_TOS_INT_r11 879 +#define _GUARD_TOS_INT_r22 880 +#define _GUARD_TOS_INT_r33 881 +#define _GUARD_TOS_LIST_r01 882 +#define _GUARD_TOS_LIST_r11 883 +#define _GUARD_TOS_LIST_r22 884 +#define _GUARD_TOS_LIST_r33 885 +#define _GUARD_TOS_OVERFLOWED_r01 886 +#define _GUARD_TOS_OVERFLOWED_r11 887 +#define _GUARD_TOS_OVERFLOWED_r22 888 +#define _GUARD_TOS_OVERFLOWED_r33 889 +#define _GUARD_TOS_SLICE_r01 890 +#define _GUARD_TOS_SLICE_r11 891 +#define _GUARD_TOS_SLICE_r22 892 +#define _GUARD_TOS_SLICE_r33 893 +#define _GUARD_TOS_TUPLE_r01 894 +#define _GUARD_TOS_TUPLE_r11 895 +#define _GUARD_TOS_TUPLE_r22 896 +#define _GUARD_TOS_TUPLE_r33 897 +#define _GUARD_TOS_UNICODE_r01 898 +#define _GUARD_TOS_UNICODE_r11 899 +#define _GUARD_TOS_UNICODE_r22 900 +#define _GUARD_TOS_UNICODE_r33 901 +#define _GUARD_TYPE_VERSION_r01 902 +#define _GUARD_TYPE_VERSION_r11 903 +#define _GUARD_TYPE_VERSION_r22 904 +#define _GUARD_TYPE_VERSION_r33 905 +#define _GUARD_TYPE_VERSION_AND_LOCK_r01 906 +#define _GUARD_TYPE_VERSION_AND_LOCK_r11 907 +#define _GUARD_TYPE_VERSION_AND_LOCK_r22 908 +#define _GUARD_TYPE_VERSION_AND_LOCK_r33 909 +#define _HANDLE_PENDING_AND_DEOPT_r00 910 +#define _HANDLE_PENDING_AND_DEOPT_r10 911 +#define _HANDLE_PENDING_AND_DEOPT_r20 912 +#define _HANDLE_PENDING_AND_DEOPT_r30 913 +#define _IMPORT_FROM_r12 914 +#define _IMPORT_NAME_r21 915 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00 916 +#define _INIT_CALL_PY_EXACT_ARGS_r01 917 +#define _INIT_CALL_PY_EXACT_ARGS_0_r01 918 +#define _INIT_CALL_PY_EXACT_ARGS_1_r01 919 +#define _INIT_CALL_PY_EXACT_ARGS_2_r01 920 +#define _INIT_CALL_PY_EXACT_ARGS_3_r01 921 +#define _INIT_CALL_PY_EXACT_ARGS_4_r01 922 +#define _INSERT_NULL_r10 923 +#define _INSTRUMENTED_FOR_ITER_r23 924 +#define _INSTRUMENTED_INSTRUCTION_r00 925 +#define _INSTRUMENTED_JUMP_FORWARD_r00 926 +#define _INSTRUMENTED_JUMP_FORWARD_r11 927 +#define _INSTRUMENTED_JUMP_FORWARD_r22 928 +#define _INSTRUMENTED_JUMP_FORWARD_r33 929 +#define _INSTRUMENTED_LINE_r00 930 +#define _INSTRUMENTED_NOT_TAKEN_r00 931 +#define _INSTRUMENTED_NOT_TAKEN_r11 932 +#define _INSTRUMENTED_NOT_TAKEN_r22 933 +#define _INSTRUMENTED_NOT_TAKEN_r33 934 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r00 935 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r10 936 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r21 937 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r32 938 +#define _INSTRUMENTED_POP_JUMP_IF_NONE_r10 939 +#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE_r10 940 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r00 941 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r10 942 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r21 943 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r32 944 +#define _IS_NONE_r11 945 +#define _IS_OP_r21 946 +#define _ITER_CHECK_LIST_r02 947 +#define _ITER_CHECK_LIST_r12 948 +#define _ITER_CHECK_LIST_r22 949 +#define _ITER_CHECK_LIST_r33 950 +#define _ITER_CHECK_RANGE_r02 951 +#define _ITER_CHECK_RANGE_r12 952 +#define _ITER_CHECK_RANGE_r22 953 +#define _ITER_CHECK_RANGE_r33 954 +#define _ITER_CHECK_TUPLE_r02 955 +#define _ITER_CHECK_TUPLE_r12 956 +#define _ITER_CHECK_TUPLE_r22 957 +#define _ITER_CHECK_TUPLE_r33 958 +#define _ITER_JUMP_LIST_r02 959 +#define _ITER_JUMP_LIST_r12 960 +#define _ITER_JUMP_LIST_r22 961 +#define _ITER_JUMP_LIST_r33 962 +#define _ITER_JUMP_RANGE_r02 963 +#define _ITER_JUMP_RANGE_r12 964 +#define _ITER_JUMP_RANGE_r22 965 +#define _ITER_JUMP_RANGE_r33 966 +#define _ITER_JUMP_TUPLE_r02 967 +#define _ITER_JUMP_TUPLE_r12 968 +#define _ITER_JUMP_TUPLE_r22 969 +#define _ITER_JUMP_TUPLE_r33 970 +#define _ITER_NEXT_LIST_r23 971 +#define _ITER_NEXT_LIST_TIER_TWO_r23 972 +#define _ITER_NEXT_RANGE_r03 973 +#define _ITER_NEXT_RANGE_r13 974 +#define _ITER_NEXT_RANGE_r23 975 +#define _ITER_NEXT_TUPLE_r03 976 +#define _ITER_NEXT_TUPLE_r13 977 +#define _ITER_NEXT_TUPLE_r23 978 +#define _JUMP_BACKWARD_NO_INTERRUPT_r00 979 +#define _JUMP_BACKWARD_NO_INTERRUPT_r11 980 +#define _JUMP_BACKWARD_NO_INTERRUPT_r22 981 +#define _JUMP_BACKWARD_NO_INTERRUPT_r33 982 +#define _JUMP_TO_TOP_r00 983 +#define _LIST_APPEND_r10 984 +#define _LIST_EXTEND_r10 985 +#define _LOAD_ATTR_r10 986 +#define _LOAD_ATTR_CLASS_r11 987 +#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_r11 988 +#define _LOAD_ATTR_INSTANCE_VALUE_r02 989 +#define _LOAD_ATTR_INSTANCE_VALUE_r12 990 +#define _LOAD_ATTR_INSTANCE_VALUE_r23 991 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r02 992 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r12 993 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r23 994 +#define _LOAD_ATTR_METHOD_NO_DICT_r02 995 +#define _LOAD_ATTR_METHOD_NO_DICT_r12 996 +#define _LOAD_ATTR_METHOD_NO_DICT_r23 997 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r02 998 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r12 999 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r23 1000 +#define _LOAD_ATTR_MODULE_r11 1001 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11 1002 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11 1003 +#define _LOAD_ATTR_PROPERTY_FRAME_r11 1004 +#define _LOAD_ATTR_SLOT_r11 1005 +#define _LOAD_ATTR_WITH_HINT_r12 1006 +#define _LOAD_BUILD_CLASS_r01 1007 +#define _LOAD_BYTECODE_r00 1008 +#define _LOAD_COMMON_CONSTANT_r01 1009 +#define _LOAD_COMMON_CONSTANT_r12 1010 +#define _LOAD_COMMON_CONSTANT_r23 1011 +#define _LOAD_CONST_r01 1012 +#define _LOAD_CONST_r12 1013 +#define _LOAD_CONST_r23 1014 +#define _LOAD_CONST_INLINE_r01 1015 +#define _LOAD_CONST_INLINE_r12 1016 +#define _LOAD_CONST_INLINE_r23 1017 +#define _LOAD_CONST_INLINE_BORROW_r01 1018 +#define _LOAD_CONST_INLINE_BORROW_r12 1019 +#define _LOAD_CONST_INLINE_BORROW_r23 1020 +#define _LOAD_CONST_UNDER_INLINE_r02 1021 +#define _LOAD_CONST_UNDER_INLINE_r12 1022 +#define _LOAD_CONST_UNDER_INLINE_r23 1023 +#define _LOAD_CONST_UNDER_INLINE_BORROW_r02 1024 +#define _LOAD_CONST_UNDER_INLINE_BORROW_r12 1025 +#define _LOAD_CONST_UNDER_INLINE_BORROW_r23 1026 +#define _LOAD_DEREF_r01 1027 +#define _LOAD_FAST_r01 1028 +#define _LOAD_FAST_r12 1029 +#define _LOAD_FAST_r23 1030 +#define _LOAD_FAST_0_r01 1031 +#define _LOAD_FAST_0_r12 1032 +#define _LOAD_FAST_0_r23 1033 +#define _LOAD_FAST_1_r01 1034 +#define _LOAD_FAST_1_r12 1035 +#define _LOAD_FAST_1_r23 1036 +#define _LOAD_FAST_2_r01 1037 +#define _LOAD_FAST_2_r12 1038 +#define _LOAD_FAST_2_r23 1039 +#define _LOAD_FAST_3_r01 1040 +#define _LOAD_FAST_3_r12 1041 +#define _LOAD_FAST_3_r23 1042 +#define _LOAD_FAST_4_r01 1043 +#define _LOAD_FAST_4_r12 1044 +#define _LOAD_FAST_4_r23 1045 +#define _LOAD_FAST_5_r01 1046 +#define _LOAD_FAST_5_r12 1047 +#define _LOAD_FAST_5_r23 1048 +#define _LOAD_FAST_6_r01 1049 +#define _LOAD_FAST_6_r12 1050 +#define _LOAD_FAST_6_r23 1051 +#define _LOAD_FAST_7_r01 1052 +#define _LOAD_FAST_7_r12 1053 +#define _LOAD_FAST_7_r23 1054 +#define _LOAD_FAST_AND_CLEAR_r01 1055 +#define _LOAD_FAST_AND_CLEAR_r12 1056 +#define _LOAD_FAST_AND_CLEAR_r23 1057 +#define _LOAD_FAST_BORROW_r01 1058 +#define _LOAD_FAST_BORROW_r12 1059 +#define _LOAD_FAST_BORROW_r23 1060 +#define _LOAD_FAST_BORROW_0_r01 1061 +#define _LOAD_FAST_BORROW_0_r12 1062 +#define _LOAD_FAST_BORROW_0_r23 1063 +#define _LOAD_FAST_BORROW_1_r01 1064 +#define _LOAD_FAST_BORROW_1_r12 1065 +#define _LOAD_FAST_BORROW_1_r23 1066 +#define _LOAD_FAST_BORROW_2_r01 1067 +#define _LOAD_FAST_BORROW_2_r12 1068 +#define _LOAD_FAST_BORROW_2_r23 1069 +#define _LOAD_FAST_BORROW_3_r01 1070 +#define _LOAD_FAST_BORROW_3_r12 1071 +#define _LOAD_FAST_BORROW_3_r23 1072 +#define _LOAD_FAST_BORROW_4_r01 1073 +#define _LOAD_FAST_BORROW_4_r12 1074 +#define _LOAD_FAST_BORROW_4_r23 1075 +#define _LOAD_FAST_BORROW_5_r01 1076 +#define _LOAD_FAST_BORROW_5_r12 1077 +#define _LOAD_FAST_BORROW_5_r23 1078 +#define _LOAD_FAST_BORROW_6_r01 1079 +#define _LOAD_FAST_BORROW_6_r12 1080 +#define _LOAD_FAST_BORROW_6_r23 1081 +#define _LOAD_FAST_BORROW_7_r01 1082 +#define _LOAD_FAST_BORROW_7_r12 1083 +#define _LOAD_FAST_BORROW_7_r23 1084 +#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r02 1085 +#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r13 1086 +#define _LOAD_FAST_CHECK_r01 1087 +#define _LOAD_FAST_CHECK_r12 1088 +#define _LOAD_FAST_CHECK_r23 1089 +#define _LOAD_FAST_LOAD_FAST_r02 1090 +#define _LOAD_FAST_LOAD_FAST_r13 1091 +#define _LOAD_FROM_DICT_OR_DEREF_r11 1092 +#define _LOAD_FROM_DICT_OR_GLOBALS_r11 1093 +#define _LOAD_GLOBAL_r00 1094 +#define _LOAD_GLOBAL_BUILTINS_r01 1095 +#define _LOAD_GLOBAL_MODULE_r01 1096 +#define _LOAD_LOCALS_r01 1097 +#define _LOAD_LOCALS_r12 1098 +#define _LOAD_LOCALS_r23 1099 +#define _LOAD_NAME_r01 1100 +#define _LOAD_SMALL_INT_r01 1101 +#define _LOAD_SMALL_INT_r12 1102 +#define _LOAD_SMALL_INT_r23 1103 +#define _LOAD_SMALL_INT_0_r01 1104 +#define _LOAD_SMALL_INT_0_r12 1105 +#define _LOAD_SMALL_INT_0_r23 1106 +#define _LOAD_SMALL_INT_1_r01 1107 +#define _LOAD_SMALL_INT_1_r12 1108 +#define _LOAD_SMALL_INT_1_r23 1109 +#define _LOAD_SMALL_INT_2_r01 1110 +#define _LOAD_SMALL_INT_2_r12 1111 +#define _LOAD_SMALL_INT_2_r23 1112 +#define _LOAD_SMALL_INT_3_r01 1113 +#define _LOAD_SMALL_INT_3_r12 1114 +#define _LOAD_SMALL_INT_3_r23 1115 +#define _LOAD_SPECIAL_r00 1116 +#define _LOAD_SUPER_ATTR_ATTR_r31 1117 +#define _LOAD_SUPER_ATTR_METHOD_r32 1118 +#define _MAKE_CALLARGS_A_TUPLE_r33 1119 +#define _MAKE_CELL_r00 1120 +#define _MAKE_FUNCTION_r11 1121 +#define _MAKE_WARM_r00 1122 +#define _MAKE_WARM_r11 1123 +#define _MAKE_WARM_r22 1124 +#define _MAKE_WARM_r33 1125 +#define _MAP_ADD_r20 1126 +#define _MATCH_CLASS_r31 1127 +#define _MATCH_KEYS_r23 1128 +#define _MATCH_MAPPING_r02 1129 +#define _MATCH_MAPPING_r12 1130 +#define _MATCH_MAPPING_r23 1131 +#define _MATCH_SEQUENCE_r02 1132 +#define _MATCH_SEQUENCE_r12 1133 +#define _MATCH_SEQUENCE_r23 1134 +#define _MAYBE_EXPAND_METHOD_r00 1135 +#define _MAYBE_EXPAND_METHOD_KW_r11 1136 +#define _MONITOR_CALL_r00 1137 +#define _MONITOR_CALL_KW_r11 1138 +#define _MONITOR_JUMP_BACKWARD_r00 1139 +#define _MONITOR_JUMP_BACKWARD_r11 1140 +#define _MONITOR_JUMP_BACKWARD_r22 1141 +#define _MONITOR_JUMP_BACKWARD_r33 1142 +#define _MONITOR_RESUME_r00 1143 +#define _NOP_r00 1144 +#define _NOP_r11 1145 +#define _NOP_r22 1146 +#define _NOP_r33 1147 +#define _POP_CALL_r20 1148 +#define _POP_CALL_LOAD_CONST_INLINE_BORROW_r21 1149 +#define _POP_CALL_ONE_r30 1150 +#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW_r31 1151 +#define _POP_CALL_TWO_r30 1152 +#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW_r31 1153 +#define _POP_EXCEPT_r10 1154 +#define _POP_ITER_r20 1155 +#define _POP_JUMP_IF_FALSE_r00 1156 +#define _POP_JUMP_IF_FALSE_r10 1157 +#define _POP_JUMP_IF_FALSE_r21 1158 +#define _POP_JUMP_IF_FALSE_r32 1159 +#define _POP_JUMP_IF_TRUE_r00 1160 +#define _POP_JUMP_IF_TRUE_r10 1161 +#define _POP_JUMP_IF_TRUE_r21 1162 +#define _POP_JUMP_IF_TRUE_r32 1163 +#define _POP_TOP_r10 1164 +#define _POP_TOP_FLOAT_r00 1165 +#define _POP_TOP_FLOAT_r10 1166 +#define _POP_TOP_FLOAT_r21 1167 +#define _POP_TOP_FLOAT_r32 1168 +#define _POP_TOP_INT_r00 1169 +#define _POP_TOP_INT_r10 1170 +#define _POP_TOP_INT_r21 1171 +#define _POP_TOP_INT_r32 1172 +#define _POP_TOP_LOAD_CONST_INLINE_r11 1173 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW_r11 1174 +#define _POP_TOP_NOP_r00 1175 +#define _POP_TOP_NOP_r10 1176 +#define _POP_TOP_NOP_r21 1177 +#define _POP_TOP_NOP_r32 1178 +#define _POP_TOP_UNICODE_r00 1179 +#define _POP_TOP_UNICODE_r10 1180 +#define _POP_TOP_UNICODE_r21 1181 +#define _POP_TOP_UNICODE_r32 1182 +#define _POP_TWO_r20 1183 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW_r21 1184 +#define _PUSH_EXC_INFO_r02 1185 +#define _PUSH_EXC_INFO_r12 1186 +#define _PUSH_EXC_INFO_r23 1187 +#define _PUSH_FRAME_r10 1188 +#define _PUSH_NULL_r01 1189 +#define _PUSH_NULL_r12 1190 +#define _PUSH_NULL_r23 1191 +#define _PUSH_NULL_CONDITIONAL_r00 1192 +#define _PY_FRAME_GENERAL_r01 1193 +#define _PY_FRAME_KW_r11 1194 +#define _QUICKEN_RESUME_r00 1195 +#define _QUICKEN_RESUME_r11 1196 +#define _QUICKEN_RESUME_r22 1197 +#define _QUICKEN_RESUME_r33 1198 +#define _REPLACE_WITH_TRUE_r11 1199 +#define _RESUME_CHECK_r00 1200 +#define _RESUME_CHECK_r11 1201 +#define _RESUME_CHECK_r22 1202 +#define _RESUME_CHECK_r33 1203 +#define _RETURN_GENERATOR_r01 1204 +#define _RETURN_VALUE_r11 1205 +#define _SAVE_RETURN_OFFSET_r00 1206 +#define _SAVE_RETURN_OFFSET_r11 1207 +#define _SAVE_RETURN_OFFSET_r22 1208 +#define _SAVE_RETURN_OFFSET_r33 1209 +#define _SEND_r22 1210 +#define _SEND_GEN_FRAME_r22 1211 +#define _SETUP_ANNOTATIONS_r00 1212 +#define _SET_ADD_r10 1213 +#define _SET_FUNCTION_ATTRIBUTE_r01 1214 +#define _SET_FUNCTION_ATTRIBUTE_r11 1215 +#define _SET_FUNCTION_ATTRIBUTE_r21 1216 +#define _SET_FUNCTION_ATTRIBUTE_r32 1217 +#define _SET_IP_r00 1218 +#define _SET_IP_r11 1219 +#define _SET_IP_r22 1220 +#define _SET_IP_r33 1221 +#define _SET_UPDATE_r10 1222 +#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r02 1223 +#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r12 1224 +#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r22 1225 +#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r32 1226 +#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03 1227 +#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13 1228 +#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23 1229 +#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33 1230 +#define _SPILL_OR_RELOAD_r01 1231 +#define _SPILL_OR_RELOAD_r02 1232 +#define _SPILL_OR_RELOAD_r03 1233 +#define _SPILL_OR_RELOAD_r10 1234 +#define _SPILL_OR_RELOAD_r12 1235 +#define _SPILL_OR_RELOAD_r13 1236 +#define _SPILL_OR_RELOAD_r20 1237 +#define _SPILL_OR_RELOAD_r21 1238 +#define _SPILL_OR_RELOAD_r23 1239 +#define _SPILL_OR_RELOAD_r30 1240 +#define _SPILL_OR_RELOAD_r31 1241 +#define _SPILL_OR_RELOAD_r32 1242 +#define _START_EXECUTOR_r00 1243 +#define _STORE_ATTR_r20 1244 +#define _STORE_ATTR_INSTANCE_VALUE_r21 1245 +#define _STORE_ATTR_SLOT_r21 1246 +#define _STORE_ATTR_WITH_HINT_r21 1247 +#define _STORE_DEREF_r10 1248 +#define _STORE_FAST_r10 1249 +#define _STORE_FAST_0_r10 1250 +#define _STORE_FAST_1_r10 1251 +#define _STORE_FAST_2_r10 1252 +#define _STORE_FAST_3_r10 1253 +#define _STORE_FAST_4_r10 1254 +#define _STORE_FAST_5_r10 1255 +#define _STORE_FAST_6_r10 1256 +#define _STORE_FAST_7_r10 1257 +#define _STORE_FAST_LOAD_FAST_r11 1258 +#define _STORE_FAST_STORE_FAST_r20 1259 +#define _STORE_GLOBAL_r10 1260 +#define _STORE_NAME_r10 1261 +#define _STORE_SLICE_r30 1262 +#define _STORE_SUBSCR_r30 1263 +#define _STORE_SUBSCR_DICT_r31 1264 +#define _STORE_SUBSCR_LIST_INT_r32 1265 +#define _SWAP_r11 1266 +#define _SWAP_2_r02 1267 +#define _SWAP_2_r12 1268 +#define _SWAP_2_r22 1269 +#define _SWAP_2_r33 1270 +#define _SWAP_3_r03 1271 +#define _SWAP_3_r13 1272 +#define _SWAP_3_r23 1273 +#define _SWAP_3_r33 1274 +#define _TIER2_RESUME_CHECK_r00 1275 +#define _TIER2_RESUME_CHECK_r11 1276 +#define _TIER2_RESUME_CHECK_r22 1277 +#define _TIER2_RESUME_CHECK_r33 1278 +#define _TO_BOOL_r11 1279 +#define _TO_BOOL_BOOL_r01 1280 +#define _TO_BOOL_BOOL_r11 1281 +#define _TO_BOOL_BOOL_r22 1282 +#define _TO_BOOL_BOOL_r33 1283 +#define _TO_BOOL_INT_r11 1284 +#define _TO_BOOL_LIST_r11 1285 +#define _TO_BOOL_NONE_r01 1286 +#define _TO_BOOL_NONE_r11 1287 +#define _TO_BOOL_NONE_r22 1288 +#define _TO_BOOL_NONE_r33 1289 +#define _TO_BOOL_STR_r11 1290 +#define _TRACE_RECORD_r00 1291 +#define _UNARY_INVERT_r11 1292 +#define _UNARY_NEGATIVE_r11 1293 +#define _UNARY_NOT_r01 1294 +#define _UNARY_NOT_r11 1295 +#define _UNARY_NOT_r22 1296 +#define _UNARY_NOT_r33 1297 +#define _UNPACK_EX_r10 1298 +#define _UNPACK_SEQUENCE_r10 1299 +#define _UNPACK_SEQUENCE_LIST_r10 1300 +#define _UNPACK_SEQUENCE_TUPLE_r10 1301 +#define _UNPACK_SEQUENCE_TWO_TUPLE_r12 1302 +#define _WITH_EXCEPT_START_r33 1303 +#define _YIELD_VALUE_r11 1304 +#define MAX_UOP_REGS_ID 1304 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 8b14ca794ce8d5..a61df5642a4d78 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -1862,12 +1862,12 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { }, }, [_COMPARE_OP_FLOAT] = { - .best = { 0, 1, 2, 3 }, + .best = { 0, 1, 2, 2 }, .entries = { - { 1, 0, _COMPARE_OP_FLOAT_r01 }, - { 1, 1, _COMPARE_OP_FLOAT_r11 }, - { 1, 2, _COMPARE_OP_FLOAT_r21 }, - { 2, 3, _COMPARE_OP_FLOAT_r32 }, + { 3, 0, _COMPARE_OP_FLOAT_r03 }, + { 3, 1, _COMPARE_OP_FLOAT_r13 }, + { 3, 2, _COMPARE_OP_FLOAT_r23 }, + { -1, -1, -1 }, }, }, [_COMPARE_OP_INT] = { @@ -1884,7 +1884,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .entries = { { -1, -1, -1 }, { -1, -1, -1 }, - { 1, 2, _COMPARE_OP_STR_r21 }, + { 3, 2, _COMPARE_OP_STR_r23 }, { -1, -1, -1 }, }, }, @@ -3568,12 +3568,11 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_STORE_ATTR_WITH_HINT_r21] = _STORE_ATTR_WITH_HINT, [_STORE_ATTR_SLOT_r21] = _STORE_ATTR_SLOT, [_COMPARE_OP_r21] = _COMPARE_OP, - [_COMPARE_OP_FLOAT_r01] = _COMPARE_OP_FLOAT, - [_COMPARE_OP_FLOAT_r11] = _COMPARE_OP_FLOAT, - [_COMPARE_OP_FLOAT_r21] = _COMPARE_OP_FLOAT, - [_COMPARE_OP_FLOAT_r32] = _COMPARE_OP_FLOAT, + [_COMPARE_OP_FLOAT_r03] = _COMPARE_OP_FLOAT, + [_COMPARE_OP_FLOAT_r13] = _COMPARE_OP_FLOAT, + [_COMPARE_OP_FLOAT_r23] = _COMPARE_OP_FLOAT, [_COMPARE_OP_INT_r23] = _COMPARE_OP_INT, - [_COMPARE_OP_STR_r21] = _COMPARE_OP_STR, + [_COMPARE_OP_STR_r23] = _COMPARE_OP_STR, [_IS_OP_r21] = _IS_OP, [_CONTAINS_OP_r21] = _CONTAINS_OP, [_GUARD_TOS_ANY_SET_r01] = _GUARD_TOS_ANY_SET, @@ -4113,14 +4112,13 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_COMPARE_OP] = "_COMPARE_OP", [_COMPARE_OP_r21] = "_COMPARE_OP_r21", [_COMPARE_OP_FLOAT] = "_COMPARE_OP_FLOAT", - [_COMPARE_OP_FLOAT_r01] = "_COMPARE_OP_FLOAT_r01", - [_COMPARE_OP_FLOAT_r11] = "_COMPARE_OP_FLOAT_r11", - [_COMPARE_OP_FLOAT_r21] = "_COMPARE_OP_FLOAT_r21", - [_COMPARE_OP_FLOAT_r32] = "_COMPARE_OP_FLOAT_r32", + [_COMPARE_OP_FLOAT_r03] = "_COMPARE_OP_FLOAT_r03", + [_COMPARE_OP_FLOAT_r13] = "_COMPARE_OP_FLOAT_r13", + [_COMPARE_OP_FLOAT_r23] = "_COMPARE_OP_FLOAT_r23", [_COMPARE_OP_INT] = "_COMPARE_OP_INT", [_COMPARE_OP_INT_r23] = "_COMPARE_OP_INT_r23", [_COMPARE_OP_STR] = "_COMPARE_OP_STR", - [_COMPARE_OP_STR_r21] = "_COMPARE_OP_STR_r21", + [_COMPARE_OP_STR_r23] = "_COMPARE_OP_STR_r23", [_CONTAINS_OP] = "_CONTAINS_OP", [_CONTAINS_OP_r21] = "_CONTAINS_OP_r21", [_CONTAINS_OP_DICT] = "_CONTAINS_OP_DICT", diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 9c9135411f9078..62fa02e10d949b 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1664,6 +1664,7 @@ def testfunc(n): self.assertNotIn("_COMPARE_OP_INT", uops) self.assertNotIn("_POP_TWO_LOAD_CONST_INLINE_BORROW", uops) + @unittest.skip("TODO (gh-142764): Re-enable after we get back automatic constant propagation.") def test_compare_op_str_pop_two_load_const_inline_borrow(self): def testfunc(n): x = 0 @@ -1681,6 +1682,7 @@ def testfunc(n): self.assertNotIn("_COMPARE_OP_STR", uops) self.assertNotIn("_POP_TWO_LOAD_CONST_INLINE_BORROW", uops) + @unittest.skip("TODO (gh-142764): Re-enable after we get back automatic constant propagation.") def test_compare_op_float_pop_two_load_const_inline_borrow(self): def testfunc(n): x = 0 @@ -2609,6 +2611,36 @@ def testfunc(n): self.assertIn("_POP_TOP_NOP", uops) self.assertNotIn("_POP_TOP", uops) + def test_float_cmp_op_refcount_elimination(self): + def testfunc(n): + c = 1.0 + res = False + for _ in range(n): + res = c == c + return res + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_COMPARE_OP_FLOAT", uops) + self.assertIn("_POP_TOP_NOP", uops) + self.assertNotIn("_POP_TOP", uops) + + def test_str_cmp_op_refcount_elimination(self): + def testfunc(n): + c = "a" + res = False + for _ in range(n): + res = c == c + return res + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_COMPARE_OP_STR", uops) + self.assertIn("_POP_TOP_NOP", uops) + self.assertNotIn("_POP_TOP", uops) + def test_unicode_add_op_refcount_elimination(self): def testfunc(n): c = "a" diff --git a/Python/bytecodes.c b/Python/bytecodes.c index b909ee187950ec..43239b129f1b4b 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2713,15 +2713,15 @@ dummy_func( macro(COMPARE_OP) = _SPECIALIZE_COMPARE_OP + _COMPARE_OP; macro(COMPARE_OP_FLOAT) = - _GUARD_TOS_FLOAT + _GUARD_NOS_FLOAT + unused/1 + _COMPARE_OP_FLOAT; + _GUARD_TOS_FLOAT + _GUARD_NOS_FLOAT + unused/1 + _COMPARE_OP_FLOAT + _POP_TOP_FLOAT + _POP_TOP_FLOAT; macro(COMPARE_OP_INT) = _GUARD_TOS_INT + _GUARD_NOS_INT + unused/1 + _COMPARE_OP_INT + _POP_TOP_INT + _POP_TOP_INT; macro(COMPARE_OP_STR) = - _GUARD_TOS_UNICODE + _GUARD_NOS_UNICODE + unused/1 + _COMPARE_OP_STR; + _GUARD_TOS_UNICODE + _GUARD_NOS_UNICODE + unused/1 + _COMPARE_OP_STR + _POP_TOP_UNICODE + _POP_TOP_UNICODE; - op(_COMPARE_OP_FLOAT, (left, right -- res)) { + op(_COMPARE_OP_FLOAT, (left, right -- res, l, r)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); @@ -2730,9 +2730,9 @@ dummy_func( double dright = PyFloat_AS_DOUBLE(right_o); // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg int sign_ish = COMPARISON_BIT(dleft, dright); - PyStackRef_CLOSE_SPECIALIZED(left, _PyFloat_ExactDealloc); + l = left; + r = right; DEAD(left); - PyStackRef_CLOSE_SPECIALIZED(right, _PyFloat_ExactDealloc); DEAD(right); res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; // It's always a bool, so we don't care about oparg & 16. @@ -2761,16 +2761,16 @@ dummy_func( } // Similar to COMPARE_OP_FLOAT, but for ==, != only - op(_COMPARE_OP_STR, (left, right -- res)) { + op(_COMPARE_OP_STR, (left, right -- res, l, r)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); STAT_INC(COMPARE_OP, hit); int eq = _PyUnicode_Equal(left_o, right_o); assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); - PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + l = left; + r = right; DEAD(left); - PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); DEAD(right); assert(eq == 0 || eq == 1); assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index b50347615c02f8..65941f25d897d9 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -8930,12 +8930,14 @@ break; } - case _COMPARE_OP_FLOAT_r01: { + case _COMPARE_OP_FLOAT_r03: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef right; _PyStackRef left; _PyStackRef res; + _PyStackRef l; + _PyStackRef r; oparg = CURRENT_OPARG(); right = stack_pointer[-1]; left = stack_pointer[-2]; @@ -8945,23 +8947,27 @@ double dleft = PyFloat_AS_DOUBLE(left_o); double dright = PyFloat_AS_DOUBLE(right_o); int sign_ish = COMPARISON_BIT(dleft, dright); - PyStackRef_CLOSE_SPECIALIZED(left, _PyFloat_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyFloat_ExactDealloc); + l = left; + r = right; res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; + _tos_cache2 = r; + _tos_cache1 = l; _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(1); + SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _COMPARE_OP_FLOAT_r11: { + case _COMPARE_OP_FLOAT_r13: { CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef right; _PyStackRef left; _PyStackRef res; + _PyStackRef l; + _PyStackRef r; _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); right = _stack_item_0; @@ -8972,23 +8978,27 @@ double dleft = PyFloat_AS_DOUBLE(left_o); double dright = PyFloat_AS_DOUBLE(right_o); int sign_ish = COMPARISON_BIT(dleft, dright); - PyStackRef_CLOSE_SPECIALIZED(left, _PyFloat_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyFloat_ExactDealloc); + l = left; + r = right; res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; + _tos_cache2 = r; + _tos_cache1 = l; _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(1); + SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _COMPARE_OP_FLOAT_r21: { + case _COMPARE_OP_FLOAT_r23: { CHECK_CURRENT_CACHED_VALUES(2); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef right; _PyStackRef left; _PyStackRef res; + _PyStackRef l; + _PyStackRef r; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; oparg = CURRENT_OPARG(); @@ -9000,39 +9010,13 @@ double dleft = PyFloat_AS_DOUBLE(left_o); double dright = PyFloat_AS_DOUBLE(right_o); int sign_ish = COMPARISON_BIT(dleft, dright); - PyStackRef_CLOSE_SPECIALIZED(left, _PyFloat_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyFloat_ExactDealloc); + l = left; + r = right; res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; + _tos_cache2 = r; + _tos_cache1 = l; _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _COMPARE_OP_FLOAT_r32: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - oparg = CURRENT_OPARG(); - right = _stack_item_2; - left = _stack_item_1; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - STAT_INC(COMPARE_OP, hit); - double dleft = PyFloat_AS_DOUBLE(left_o); - double dright = PyFloat_AS_DOUBLE(right_o); - int sign_ish = COMPARISON_BIT(dleft, dright); - PyStackRef_CLOSE_SPECIALIZED(left, _PyFloat_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyFloat_ExactDealloc); - res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; - _tos_cache1 = res; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); + SET_CURRENT_CACHED_VALUES(3); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } @@ -9071,12 +9055,14 @@ break; } - case _COMPARE_OP_STR_r21: { + case _COMPARE_OP_STR_r23: { CHECK_CURRENT_CACHED_VALUES(2); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef right; _PyStackRef left; _PyStackRef res; + _PyStackRef l; + _PyStackRef r; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; oparg = CURRENT_OPARG(); @@ -9087,16 +9073,16 @@ STAT_INC(COMPARE_OP, hit); int eq = _PyUnicode_Equal(left_o, right_o); assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); - PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + l = left; + r = right; assert(eq == 0 || eq == 1); assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; + _tos_cache2 = r; + _tos_cache1 = l; _tos_cache0 = res; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); + SET_CURRENT_CACHED_VALUES(3); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 50759933814c4f..1a91cdaf6537e7 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4449,6 +4449,8 @@ _PyStackRef left; _PyStackRef right; _PyStackRef res; + _PyStackRef l; + _PyStackRef r; // _GUARD_TOS_FLOAT { value = stack_pointer[-1]; @@ -4479,10 +4481,22 @@ double dleft = PyFloat_AS_DOUBLE(left_o); double dright = PyFloat_AS_DOUBLE(right_o); int sign_ish = COMPARISON_BIT(dleft, dright); - PyStackRef_CLOSE_SPECIALIZED(left, _PyFloat_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyFloat_ExactDealloc); + l = left; + r = right; res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; } + // _POP_TOP_FLOAT + { + value = r; + assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); + } + // _POP_TOP_FLOAT + { + value = l; + assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); + } stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -4578,6 +4592,8 @@ _PyStackRef left; _PyStackRef right; _PyStackRef res; + _PyStackRef l; + _PyStackRef r; // _GUARD_TOS_UNICODE { value = stack_pointer[-1]; @@ -4608,13 +4624,25 @@ STAT_INC(COMPARE_OP, hit); int eq = _PyUnicode_Equal(left_o, right_o); assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); - PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + l = left; + r = right; assert(eq == 0 || eq == 1); assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; } + // _POP_TOP_UNICODE + { + value = r; + assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); + } + // _POP_TOP_UNICODE + { + value = l; + assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); + PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); + } stack_pointer[-2] = res; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 1575b9e38424c5..9f7b2663dacfab 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -466,14 +466,16 @@ dummy_func(void) { r = right; } - op(_COMPARE_OP_FLOAT, (left, right -- res)) { - REPLACE_OPCODE_IF_EVALUATES_PURE(left, right); + op(_COMPARE_OP_FLOAT, (left, right -- res, l, r)) { res = sym_new_type(ctx, &PyBool_Type); + l = left; + r = right; } - op(_COMPARE_OP_STR, (left, right -- res)) { - REPLACE_OPCODE_IF_EVALUATES_PURE(left, right); + op(_COMPARE_OP_STR, (left, right -- res, l, r)) { res = sym_new_type(ctx, &PyBool_Type); + l = left; + r = right; } op(_IS_OP, (left, right -- b)) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index a8b043fbc875d2..630dc1703f9ab5 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1818,46 +1818,18 @@ JitOptRef right; JitOptRef left; JitOptRef res; + JitOptRef l; + JitOptRef r; right = stack_pointer[-1]; left = stack_pointer[-2]; - if ( - sym_is_safe_const(ctx, left) && - sym_is_safe_const(ctx, right) - ) { - JitOptRef left_sym = left; - JitOptRef right_sym = right; - _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); - _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); - _PyStackRef res_stackref; - /* Start of uop copied from bytecodes for constant evaluation */ - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - STAT_INC(COMPARE_OP, hit); - double dleft = PyFloat_AS_DOUBLE(left_o); - double dright = PyFloat_AS_DOUBLE(right_o); - int sign_ish = COMPARISON_BIT(dleft, dright); - PyStackRef_CLOSE_SPECIALIZED(left, _PyFloat_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyFloat_ExactDealloc); - res_stackref = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; - /* End of uop copied from bytecodes for constant evaluation */ - res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - } - } - CHECK_STACK_BOUNDS(-1); - stack_pointer[-2] = res; - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - break; - } res = sym_new_type(ctx, &PyBool_Type); - CHECK_STACK_BOUNDS(-1); + l = left; + r = right; + CHECK_STACK_BOUNDS(1); stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -1886,48 +1858,18 @@ JitOptRef right; JitOptRef left; JitOptRef res; + JitOptRef l; + JitOptRef r; right = stack_pointer[-1]; left = stack_pointer[-2]; - if ( - sym_is_safe_const(ctx, left) && - sym_is_safe_const(ctx, right) - ) { - JitOptRef left_sym = left; - JitOptRef right_sym = right; - _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); - _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); - _PyStackRef res_stackref; - /* Start of uop copied from bytecodes for constant evaluation */ - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - STAT_INC(COMPARE_OP, hit); - int eq = _PyUnicode_Equal(left_o, right_o); - assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); - PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); - assert(eq == 0 || eq == 1); - assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); - assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); - res_stackref = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; - /* End of uop copied from bytecodes for constant evaluation */ - res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { - PyObject *result = sym_get_const(ctx, res); - if (_Py_IsImmortal(result)) { - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - } - } - CHECK_STACK_BOUNDS(-1); - stack_pointer[-2] = res; - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - break; - } res = sym_new_type(ctx, &PyBool_Type); - CHECK_STACK_BOUNDS(-1); + l = left; + r = right; + CHECK_STACK_BOUNDS(1); stack_pointer[-2] = res; - stack_pointer += -1; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } From a1c630834649d54ffbca34f79650b5adcafcde71 Mon Sep 17 00:00:00 2001 From: Hai Zhu <35182391+cocolato@users.noreply.github.com> Date: Sat, 27 Dec 2025 04:30:02 +0800 Subject: [PATCH 581/638] gh-134584: Eliminate redundant refcounting from `IS_OP` (GH-143171) Eliminate redundant refcounting from IS_OP --- Include/internal/pycore_opcode_metadata.h | 2 +- Include/internal/pycore_uop_ids.h | 1746 +++++++++++---------- Include/internal/pycore_uop_metadata.h | 18 +- Lib/test/test_capi/test_opt.py | 65 + Python/bytecodes.c | 8 +- Python/executor_cases.c.h | 74 +- Python/generated_cases.c.h | 49 +- Python/optimizer_bytecodes.c | 4 +- Python/optimizer_cases.c.h | 14 +- 9 files changed, 1061 insertions(+), 919 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 7139b5a9e54483..cd2475cd2374e8 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1416,7 +1416,7 @@ _PyOpcode_macro_expansion[256] = { [GET_YIELD_FROM_ITER] = { .nuops = 1, .uops = { { _GET_YIELD_FROM_ITER, OPARG_SIMPLE, 0 } } }, [IMPORT_FROM] = { .nuops = 1, .uops = { { _IMPORT_FROM, OPARG_SIMPLE, 0 } } }, [IMPORT_NAME] = { .nuops = 1, .uops = { { _IMPORT_NAME, OPARG_SIMPLE, 0 } } }, - [IS_OP] = { .nuops = 1, .uops = { { _IS_OP, OPARG_SIMPLE, 0 } } }, + [IS_OP] = { .nuops = 3, .uops = { { _IS_OP, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [JUMP_BACKWARD] = { .nuops = 2, .uops = { { _CHECK_PERIODIC, OPARG_SIMPLE, 1 }, { _JUMP_BACKWARD_NO_INTERRUPT, OPARG_REPLACED, 1 } } }, [JUMP_BACKWARD_NO_INTERRUPT] = { .nuops = 1, .uops = { { _JUMP_BACKWARD_NO_INTERRUPT, OPARG_REPLACED, 0 } } }, [JUMP_BACKWARD_NO_JIT] = { .nuops = 2, .uops = { { _CHECK_PERIODIC, OPARG_SIMPLE, 1 }, { _JUMP_BACKWARD_NO_INTERRUPT, OPARG_REPLACED, 1 } } }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 9b3c5fbcf8d4e0..b146c4ea39b4a9 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -189,927 +189,929 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE #define _IS_NONE 435 -#define _IS_OP IS_OP -#define _ITER_CHECK_LIST 436 -#define _ITER_CHECK_RANGE 437 -#define _ITER_CHECK_TUPLE 438 -#define _ITER_JUMP_LIST 439 -#define _ITER_JUMP_RANGE 440 -#define _ITER_JUMP_TUPLE 441 -#define _ITER_NEXT_LIST 442 -#define _ITER_NEXT_LIST_TIER_TWO 443 -#define _ITER_NEXT_RANGE 444 -#define _ITER_NEXT_TUPLE 445 +#define _IS_OP 436 +#define _ITER_CHECK_LIST 437 +#define _ITER_CHECK_RANGE 438 +#define _ITER_CHECK_TUPLE 439 +#define _ITER_JUMP_LIST 440 +#define _ITER_JUMP_RANGE 441 +#define _ITER_JUMP_TUPLE 442 +#define _ITER_NEXT_LIST 443 +#define _ITER_NEXT_LIST_TIER_TWO 444 +#define _ITER_NEXT_RANGE 445 +#define _ITER_NEXT_TUPLE 446 #define _JUMP_BACKWARD_NO_INTERRUPT JUMP_BACKWARD_NO_INTERRUPT -#define _JUMP_TO_TOP 446 +#define _JUMP_TO_TOP 447 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 447 -#define _LOAD_ATTR_CLASS 448 +#define _LOAD_ATTR 448 +#define _LOAD_ATTR_CLASS 449 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 449 -#define _LOAD_ATTR_METHOD_LAZY_DICT 450 -#define _LOAD_ATTR_METHOD_NO_DICT 451 -#define _LOAD_ATTR_METHOD_WITH_VALUES 452 -#define _LOAD_ATTR_MODULE 453 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 454 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 455 -#define _LOAD_ATTR_PROPERTY_FRAME 456 -#define _LOAD_ATTR_SLOT 457 -#define _LOAD_ATTR_WITH_HINT 458 +#define _LOAD_ATTR_INSTANCE_VALUE 450 +#define _LOAD_ATTR_METHOD_LAZY_DICT 451 +#define _LOAD_ATTR_METHOD_NO_DICT 452 +#define _LOAD_ATTR_METHOD_WITH_VALUES 453 +#define _LOAD_ATTR_MODULE 454 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 455 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 456 +#define _LOAD_ATTR_PROPERTY_FRAME 457 +#define _LOAD_ATTR_SLOT 458 +#define _LOAD_ATTR_WITH_HINT 459 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 459 +#define _LOAD_BYTECODE 460 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 460 -#define _LOAD_CONST_INLINE_BORROW 461 -#define _LOAD_CONST_UNDER_INLINE 462 -#define _LOAD_CONST_UNDER_INLINE_BORROW 463 +#define _LOAD_CONST_INLINE 461 +#define _LOAD_CONST_INLINE_BORROW 462 +#define _LOAD_CONST_UNDER_INLINE 463 +#define _LOAD_CONST_UNDER_INLINE_BORROW 464 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 464 -#define _LOAD_FAST_0 465 -#define _LOAD_FAST_1 466 -#define _LOAD_FAST_2 467 -#define _LOAD_FAST_3 468 -#define _LOAD_FAST_4 469 -#define _LOAD_FAST_5 470 -#define _LOAD_FAST_6 471 -#define _LOAD_FAST_7 472 +#define _LOAD_FAST 465 +#define _LOAD_FAST_0 466 +#define _LOAD_FAST_1 467 +#define _LOAD_FAST_2 468 +#define _LOAD_FAST_3 469 +#define _LOAD_FAST_4 470 +#define _LOAD_FAST_5 471 +#define _LOAD_FAST_6 472 +#define _LOAD_FAST_7 473 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_BORROW 473 -#define _LOAD_FAST_BORROW_0 474 -#define _LOAD_FAST_BORROW_1 475 -#define _LOAD_FAST_BORROW_2 476 -#define _LOAD_FAST_BORROW_3 477 -#define _LOAD_FAST_BORROW_4 478 -#define _LOAD_FAST_BORROW_5 479 -#define _LOAD_FAST_BORROW_6 480 -#define _LOAD_FAST_BORROW_7 481 +#define _LOAD_FAST_BORROW 474 +#define _LOAD_FAST_BORROW_0 475 +#define _LOAD_FAST_BORROW_1 476 +#define _LOAD_FAST_BORROW_2 477 +#define _LOAD_FAST_BORROW_3 478 +#define _LOAD_FAST_BORROW_4 479 +#define _LOAD_FAST_BORROW_5 480 +#define _LOAD_FAST_BORROW_6 481 +#define _LOAD_FAST_BORROW_7 482 #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 482 -#define _LOAD_GLOBAL_BUILTINS 483 -#define _LOAD_GLOBAL_MODULE 484 +#define _LOAD_GLOBAL 483 +#define _LOAD_GLOBAL_BUILTINS 484 +#define _LOAD_GLOBAL_MODULE 485 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 485 -#define _LOAD_SMALL_INT_0 486 -#define _LOAD_SMALL_INT_1 487 -#define _LOAD_SMALL_INT_2 488 -#define _LOAD_SMALL_INT_3 489 -#define _LOAD_SPECIAL 490 +#define _LOAD_SMALL_INT 486 +#define _LOAD_SMALL_INT_0 487 +#define _LOAD_SMALL_INT_1 488 +#define _LOAD_SMALL_INT_2 489 +#define _LOAD_SMALL_INT_3 490 +#define _LOAD_SPECIAL 491 #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 491 +#define _MAKE_CALLARGS_A_TUPLE 492 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 492 +#define _MAKE_WARM 493 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 493 -#define _MAYBE_EXPAND_METHOD_KW 494 -#define _MONITOR_CALL 495 -#define _MONITOR_CALL_KW 496 -#define _MONITOR_JUMP_BACKWARD 497 -#define _MONITOR_RESUME 498 +#define _MAYBE_EXPAND_METHOD 494 +#define _MAYBE_EXPAND_METHOD_KW 495 +#define _MONITOR_CALL 496 +#define _MONITOR_CALL_KW 497 +#define _MONITOR_JUMP_BACKWARD 498 +#define _MONITOR_RESUME 499 #define _NOP NOP -#define _POP_CALL 499 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW 500 -#define _POP_CALL_ONE 501 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 502 -#define _POP_CALL_TWO 503 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 504 +#define _POP_CALL 500 +#define _POP_CALL_LOAD_CONST_INLINE_BORROW 501 +#define _POP_CALL_ONE 502 +#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 503 +#define _POP_CALL_TWO 504 +#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 505 #define _POP_EXCEPT POP_EXCEPT #define _POP_ITER POP_ITER -#define _POP_JUMP_IF_FALSE 505 -#define _POP_JUMP_IF_TRUE 506 +#define _POP_JUMP_IF_FALSE 506 +#define _POP_JUMP_IF_TRUE 507 #define _POP_TOP POP_TOP -#define _POP_TOP_FLOAT 507 -#define _POP_TOP_INT 508 -#define _POP_TOP_LOAD_CONST_INLINE 509 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 510 -#define _POP_TOP_NOP 511 -#define _POP_TOP_UNICODE 512 -#define _POP_TWO 513 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW 514 +#define _POP_TOP_FLOAT 508 +#define _POP_TOP_INT 509 +#define _POP_TOP_LOAD_CONST_INLINE 510 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 511 +#define _POP_TOP_NOP 512 +#define _POP_TOP_UNICODE 513 +#define _POP_TWO 514 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW 515 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 515 +#define _PUSH_FRAME 516 #define _PUSH_NULL PUSH_NULL -#define _PUSH_NULL_CONDITIONAL 516 -#define _PY_FRAME_GENERAL 517 -#define _PY_FRAME_KW 518 -#define _QUICKEN_RESUME 519 -#define _REPLACE_WITH_TRUE 520 +#define _PUSH_NULL_CONDITIONAL 517 +#define _PY_FRAME_GENERAL 518 +#define _PY_FRAME_KW 519 +#define _QUICKEN_RESUME 520 +#define _REPLACE_WITH_TRUE 521 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 521 -#define _SEND 522 -#define _SEND_GEN_FRAME 523 +#define _SAVE_RETURN_OFFSET 522 +#define _SEND 523 +#define _SEND_GEN_FRAME 524 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW 524 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW 525 -#define _SPILL_OR_RELOAD 526 -#define _START_EXECUTOR 527 -#define _STORE_ATTR 528 -#define _STORE_ATTR_INSTANCE_VALUE 529 -#define _STORE_ATTR_SLOT 530 -#define _STORE_ATTR_WITH_HINT 531 +#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW 525 +#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW 526 +#define _SPILL_OR_RELOAD 527 +#define _START_EXECUTOR 528 +#define _STORE_ATTR 529 +#define _STORE_ATTR_INSTANCE_VALUE 530 +#define _STORE_ATTR_SLOT 531 +#define _STORE_ATTR_WITH_HINT 532 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 532 -#define _STORE_FAST_0 533 -#define _STORE_FAST_1 534 -#define _STORE_FAST_2 535 -#define _STORE_FAST_3 536 -#define _STORE_FAST_4 537 -#define _STORE_FAST_5 538 -#define _STORE_FAST_6 539 -#define _STORE_FAST_7 540 +#define _STORE_FAST 533 +#define _STORE_FAST_0 534 +#define _STORE_FAST_1 535 +#define _STORE_FAST_2 536 +#define _STORE_FAST_3 537 +#define _STORE_FAST_4 538 +#define _STORE_FAST_5 539 +#define _STORE_FAST_6 540 +#define _STORE_FAST_7 541 #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 541 -#define _STORE_SUBSCR 542 -#define _STORE_SUBSCR_DICT 543 -#define _STORE_SUBSCR_LIST_INT 544 -#define _SWAP 545 -#define _SWAP_2 546 -#define _SWAP_3 547 -#define _TIER2_RESUME_CHECK 548 -#define _TO_BOOL 549 +#define _STORE_SLICE 542 +#define _STORE_SUBSCR 543 +#define _STORE_SUBSCR_DICT 544 +#define _STORE_SUBSCR_LIST_INT 545 +#define _SWAP 546 +#define _SWAP_2 547 +#define _SWAP_3 548 +#define _TIER2_RESUME_CHECK 549 +#define _TO_BOOL 550 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT -#define _TO_BOOL_LIST 550 +#define _TO_BOOL_LIST 551 #define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR 551 +#define _TO_BOOL_STR 552 #define _TRACE_RECORD TRACE_RECORD #define _UNARY_INVERT UNARY_INVERT #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 552 -#define _UNPACK_SEQUENCE_LIST 553 -#define _UNPACK_SEQUENCE_TUPLE 554 -#define _UNPACK_SEQUENCE_TWO_TUPLE 555 +#define _UNPACK_SEQUENCE 553 +#define _UNPACK_SEQUENCE_LIST 554 +#define _UNPACK_SEQUENCE_TUPLE 555 +#define _UNPACK_SEQUENCE_TWO_TUPLE 556 #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 555 -#define _BINARY_OP_r21 556 -#define _BINARY_OP_ADD_FLOAT_r03 557 -#define _BINARY_OP_ADD_FLOAT_r13 558 -#define _BINARY_OP_ADD_FLOAT_r23 559 -#define _BINARY_OP_ADD_INT_r03 560 -#define _BINARY_OP_ADD_INT_r13 561 -#define _BINARY_OP_ADD_INT_r23 562 -#define _BINARY_OP_ADD_UNICODE_r03 563 -#define _BINARY_OP_ADD_UNICODE_r13 564 -#define _BINARY_OP_ADD_UNICODE_r23 565 -#define _BINARY_OP_EXTEND_r21 566 -#define _BINARY_OP_INPLACE_ADD_UNICODE_r21 567 -#define _BINARY_OP_MULTIPLY_FLOAT_r03 568 -#define _BINARY_OP_MULTIPLY_FLOAT_r13 569 -#define _BINARY_OP_MULTIPLY_FLOAT_r23 570 -#define _BINARY_OP_MULTIPLY_INT_r03 571 -#define _BINARY_OP_MULTIPLY_INT_r13 572 -#define _BINARY_OP_MULTIPLY_INT_r23 573 -#define _BINARY_OP_SUBSCR_CHECK_FUNC_r23 574 -#define _BINARY_OP_SUBSCR_DICT_r21 575 -#define _BINARY_OP_SUBSCR_INIT_CALL_r01 576 -#define _BINARY_OP_SUBSCR_INIT_CALL_r11 577 -#define _BINARY_OP_SUBSCR_INIT_CALL_r21 578 -#define _BINARY_OP_SUBSCR_INIT_CALL_r31 579 -#define _BINARY_OP_SUBSCR_LIST_INT_r23 580 -#define _BINARY_OP_SUBSCR_LIST_SLICE_r21 581 -#define _BINARY_OP_SUBSCR_STR_INT_r23 582 -#define _BINARY_OP_SUBSCR_TUPLE_INT_r23 583 -#define _BINARY_OP_SUBTRACT_FLOAT_r03 584 -#define _BINARY_OP_SUBTRACT_FLOAT_r13 585 -#define _BINARY_OP_SUBTRACT_FLOAT_r23 586 -#define _BINARY_OP_SUBTRACT_INT_r03 587 -#define _BINARY_OP_SUBTRACT_INT_r13 588 -#define _BINARY_OP_SUBTRACT_INT_r23 589 -#define _BINARY_SLICE_r31 590 -#define _BUILD_INTERPOLATION_r01 591 -#define _BUILD_LIST_r01 592 -#define _BUILD_MAP_r01 593 -#define _BUILD_SET_r01 594 -#define _BUILD_SLICE_r01 595 -#define _BUILD_STRING_r01 596 -#define _BUILD_TEMPLATE_r21 597 -#define _BUILD_TUPLE_r01 598 -#define _CALL_BUILTIN_CLASS_r01 599 -#define _CALL_BUILTIN_FAST_r01 600 -#define _CALL_BUILTIN_FAST_WITH_KEYWORDS_r01 601 -#define _CALL_BUILTIN_O_r03 602 -#define _CALL_INTRINSIC_1_r11 603 -#define _CALL_INTRINSIC_2_r21 604 -#define _CALL_ISINSTANCE_r31 605 -#define _CALL_KW_NON_PY_r11 606 -#define _CALL_LEN_r33 607 -#define _CALL_LIST_APPEND_r03 608 -#define _CALL_LIST_APPEND_r13 609 -#define _CALL_LIST_APPEND_r23 610 -#define _CALL_LIST_APPEND_r33 611 -#define _CALL_METHOD_DESCRIPTOR_FAST_r01 612 -#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r01 613 -#define _CALL_METHOD_DESCRIPTOR_NOARGS_r01 614 -#define _CALL_METHOD_DESCRIPTOR_O_r01 615 -#define _CALL_NON_PY_GENERAL_r01 616 -#define _CALL_STR_1_r32 617 -#define _CALL_TUPLE_1_r32 618 -#define _CALL_TYPE_1_r02 619 -#define _CALL_TYPE_1_r12 620 -#define _CALL_TYPE_1_r22 621 -#define _CALL_TYPE_1_r32 622 -#define _CHECK_AND_ALLOCATE_OBJECT_r00 623 -#define _CHECK_ATTR_CLASS_r01 624 -#define _CHECK_ATTR_CLASS_r11 625 -#define _CHECK_ATTR_CLASS_r22 626 -#define _CHECK_ATTR_CLASS_r33 627 -#define _CHECK_ATTR_METHOD_LAZY_DICT_r01 628 -#define _CHECK_ATTR_METHOD_LAZY_DICT_r11 629 -#define _CHECK_ATTR_METHOD_LAZY_DICT_r22 630 -#define _CHECK_ATTR_METHOD_LAZY_DICT_r33 631 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00 632 -#define _CHECK_EG_MATCH_r22 633 -#define _CHECK_EXC_MATCH_r22 634 -#define _CHECK_FUNCTION_EXACT_ARGS_r00 635 -#define _CHECK_FUNCTION_VERSION_r00 636 -#define _CHECK_FUNCTION_VERSION_INLINE_r00 637 -#define _CHECK_FUNCTION_VERSION_INLINE_r11 638 -#define _CHECK_FUNCTION_VERSION_INLINE_r22 639 -#define _CHECK_FUNCTION_VERSION_INLINE_r33 640 -#define _CHECK_FUNCTION_VERSION_KW_r11 641 -#define _CHECK_IS_NOT_PY_CALLABLE_r00 642 -#define _CHECK_IS_NOT_PY_CALLABLE_KW_r11 643 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r01 644 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r11 645 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r22 646 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r33 647 -#define _CHECK_METHOD_VERSION_r00 648 -#define _CHECK_METHOD_VERSION_KW_r11 649 -#define _CHECK_PEP_523_r00 650 -#define _CHECK_PEP_523_r11 651 -#define _CHECK_PEP_523_r22 652 -#define _CHECK_PEP_523_r33 653 -#define _CHECK_PERIODIC_r00 654 -#define _CHECK_PERIODIC_AT_END_r00 655 -#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00 656 -#define _CHECK_RECURSION_REMAINING_r00 657 -#define _CHECK_RECURSION_REMAINING_r11 658 -#define _CHECK_RECURSION_REMAINING_r22 659 -#define _CHECK_RECURSION_REMAINING_r33 660 -#define _CHECK_STACK_SPACE_r00 661 -#define _CHECK_STACK_SPACE_OPERAND_r00 662 -#define _CHECK_STACK_SPACE_OPERAND_r11 663 -#define _CHECK_STACK_SPACE_OPERAND_r22 664 -#define _CHECK_STACK_SPACE_OPERAND_r33 665 -#define _CHECK_VALIDITY_r00 666 -#define _CHECK_VALIDITY_r11 667 -#define _CHECK_VALIDITY_r22 668 -#define _CHECK_VALIDITY_r33 669 -#define _COLD_DYNAMIC_EXIT_r00 670 -#define _COLD_EXIT_r00 671 -#define _COMPARE_OP_r21 672 -#define _COMPARE_OP_FLOAT_r03 673 -#define _COMPARE_OP_FLOAT_r13 674 -#define _COMPARE_OP_FLOAT_r23 675 -#define _COMPARE_OP_INT_r23 676 -#define _COMPARE_OP_STR_r23 677 -#define _CONTAINS_OP_r21 678 -#define _CONTAINS_OP_DICT_r21 679 -#define _CONTAINS_OP_SET_r21 680 -#define _CONVERT_VALUE_r11 681 -#define _COPY_r01 682 -#define _COPY_1_r02 683 -#define _COPY_1_r12 684 -#define _COPY_1_r23 685 -#define _COPY_2_r03 686 -#define _COPY_2_r13 687 -#define _COPY_2_r23 688 -#define _COPY_3_r03 689 -#define _COPY_3_r13 690 -#define _COPY_3_r23 691 -#define _COPY_3_r33 692 -#define _COPY_FREE_VARS_r00 693 -#define _COPY_FREE_VARS_r11 694 -#define _COPY_FREE_VARS_r22 695 -#define _COPY_FREE_VARS_r33 696 -#define _CREATE_INIT_FRAME_r01 697 -#define _DELETE_ATTR_r10 698 -#define _DELETE_DEREF_r00 699 -#define _DELETE_FAST_r00 700 -#define _DELETE_GLOBAL_r00 701 -#define _DELETE_NAME_r00 702 -#define _DELETE_SUBSCR_r20 703 -#define _DEOPT_r00 704 -#define _DEOPT_r10 705 -#define _DEOPT_r20 706 -#define _DEOPT_r30 707 -#define _DICT_MERGE_r10 708 -#define _DICT_UPDATE_r10 709 -#define _DO_CALL_r01 710 -#define _DO_CALL_FUNCTION_EX_r31 711 -#define _DO_CALL_KW_r11 712 -#define _DYNAMIC_EXIT_r00 713 -#define _DYNAMIC_EXIT_r10 714 -#define _DYNAMIC_EXIT_r20 715 -#define _DYNAMIC_EXIT_r30 716 -#define _END_FOR_r10 717 -#define _END_SEND_r21 718 -#define _ERROR_POP_N_r00 719 -#define _EXIT_INIT_CHECK_r10 720 -#define _EXIT_TRACE_r00 721 -#define _EXIT_TRACE_r10 722 -#define _EXIT_TRACE_r20 723 -#define _EXIT_TRACE_r30 724 -#define _EXPAND_METHOD_r00 725 -#define _EXPAND_METHOD_KW_r11 726 -#define _FATAL_ERROR_r00 727 -#define _FATAL_ERROR_r11 728 -#define _FATAL_ERROR_r22 729 -#define _FATAL_ERROR_r33 730 -#define _FORMAT_SIMPLE_r11 731 -#define _FORMAT_WITH_SPEC_r21 732 -#define _FOR_ITER_r23 733 -#define _FOR_ITER_GEN_FRAME_r03 734 -#define _FOR_ITER_GEN_FRAME_r13 735 -#define _FOR_ITER_GEN_FRAME_r23 736 -#define _FOR_ITER_TIER_TWO_r23 737 -#define _GET_AITER_r11 738 -#define _GET_ANEXT_r12 739 -#define _GET_AWAITABLE_r11 740 -#define _GET_ITER_r12 741 -#define _GET_LEN_r12 742 -#define _GET_YIELD_FROM_ITER_r11 743 -#define _GUARD_BINARY_OP_EXTEND_r22 744 -#define _GUARD_CALLABLE_ISINSTANCE_r03 745 -#define _GUARD_CALLABLE_ISINSTANCE_r13 746 -#define _GUARD_CALLABLE_ISINSTANCE_r23 747 -#define _GUARD_CALLABLE_ISINSTANCE_r33 748 -#define _GUARD_CALLABLE_LEN_r03 749 -#define _GUARD_CALLABLE_LEN_r13 750 -#define _GUARD_CALLABLE_LEN_r23 751 -#define _GUARD_CALLABLE_LEN_r33 752 -#define _GUARD_CALLABLE_LIST_APPEND_r03 753 -#define _GUARD_CALLABLE_LIST_APPEND_r13 754 -#define _GUARD_CALLABLE_LIST_APPEND_r23 755 -#define _GUARD_CALLABLE_LIST_APPEND_r33 756 -#define _GUARD_CALLABLE_STR_1_r03 757 -#define _GUARD_CALLABLE_STR_1_r13 758 -#define _GUARD_CALLABLE_STR_1_r23 759 -#define _GUARD_CALLABLE_STR_1_r33 760 -#define _GUARD_CALLABLE_TUPLE_1_r03 761 -#define _GUARD_CALLABLE_TUPLE_1_r13 762 -#define _GUARD_CALLABLE_TUPLE_1_r23 763 -#define _GUARD_CALLABLE_TUPLE_1_r33 764 -#define _GUARD_CALLABLE_TYPE_1_r03 765 -#define _GUARD_CALLABLE_TYPE_1_r13 766 -#define _GUARD_CALLABLE_TYPE_1_r23 767 -#define _GUARD_CALLABLE_TYPE_1_r33 768 -#define _GUARD_DORV_NO_DICT_r01 769 -#define _GUARD_DORV_NO_DICT_r11 770 -#define _GUARD_DORV_NO_DICT_r22 771 -#define _GUARD_DORV_NO_DICT_r33 772 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r01 773 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11 774 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22 775 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33 776 -#define _GUARD_GLOBALS_VERSION_r00 777 -#define _GUARD_GLOBALS_VERSION_r11 778 -#define _GUARD_GLOBALS_VERSION_r22 779 -#define _GUARD_GLOBALS_VERSION_r33 780 -#define _GUARD_IP_RETURN_GENERATOR_r00 781 -#define _GUARD_IP_RETURN_GENERATOR_r11 782 -#define _GUARD_IP_RETURN_GENERATOR_r22 783 -#define _GUARD_IP_RETURN_GENERATOR_r33 784 -#define _GUARD_IP_RETURN_VALUE_r00 785 -#define _GUARD_IP_RETURN_VALUE_r11 786 -#define _GUARD_IP_RETURN_VALUE_r22 787 -#define _GUARD_IP_RETURN_VALUE_r33 788 -#define _GUARD_IP_YIELD_VALUE_r00 789 -#define _GUARD_IP_YIELD_VALUE_r11 790 -#define _GUARD_IP_YIELD_VALUE_r22 791 -#define _GUARD_IP_YIELD_VALUE_r33 792 -#define _GUARD_IP__PUSH_FRAME_r00 793 -#define _GUARD_IP__PUSH_FRAME_r11 794 -#define _GUARD_IP__PUSH_FRAME_r22 795 -#define _GUARD_IP__PUSH_FRAME_r33 796 -#define _GUARD_IS_FALSE_POP_r00 797 -#define _GUARD_IS_FALSE_POP_r10 798 -#define _GUARD_IS_FALSE_POP_r21 799 -#define _GUARD_IS_FALSE_POP_r32 800 -#define _GUARD_IS_NONE_POP_r00 801 -#define _GUARD_IS_NONE_POP_r10 802 -#define _GUARD_IS_NONE_POP_r21 803 -#define _GUARD_IS_NONE_POP_r32 804 -#define _GUARD_IS_NOT_NONE_POP_r10 805 -#define _GUARD_IS_TRUE_POP_r00 806 -#define _GUARD_IS_TRUE_POP_r10 807 -#define _GUARD_IS_TRUE_POP_r21 808 -#define _GUARD_IS_TRUE_POP_r32 809 -#define _GUARD_KEYS_VERSION_r01 810 -#define _GUARD_KEYS_VERSION_r11 811 -#define _GUARD_KEYS_VERSION_r22 812 -#define _GUARD_KEYS_VERSION_r33 813 -#define _GUARD_NOS_DICT_r02 814 -#define _GUARD_NOS_DICT_r12 815 -#define _GUARD_NOS_DICT_r22 816 -#define _GUARD_NOS_DICT_r33 817 -#define _GUARD_NOS_FLOAT_r02 818 -#define _GUARD_NOS_FLOAT_r12 819 -#define _GUARD_NOS_FLOAT_r22 820 -#define _GUARD_NOS_FLOAT_r33 821 -#define _GUARD_NOS_INT_r02 822 -#define _GUARD_NOS_INT_r12 823 -#define _GUARD_NOS_INT_r22 824 -#define _GUARD_NOS_INT_r33 825 -#define _GUARD_NOS_LIST_r02 826 -#define _GUARD_NOS_LIST_r12 827 -#define _GUARD_NOS_LIST_r22 828 -#define _GUARD_NOS_LIST_r33 829 -#define _GUARD_NOS_NOT_NULL_r02 830 -#define _GUARD_NOS_NOT_NULL_r12 831 -#define _GUARD_NOS_NOT_NULL_r22 832 -#define _GUARD_NOS_NOT_NULL_r33 833 -#define _GUARD_NOS_NULL_r02 834 -#define _GUARD_NOS_NULL_r12 835 -#define _GUARD_NOS_NULL_r22 836 -#define _GUARD_NOS_NULL_r33 837 -#define _GUARD_NOS_OVERFLOWED_r02 838 -#define _GUARD_NOS_OVERFLOWED_r12 839 -#define _GUARD_NOS_OVERFLOWED_r22 840 -#define _GUARD_NOS_OVERFLOWED_r33 841 -#define _GUARD_NOS_TUPLE_r02 842 -#define _GUARD_NOS_TUPLE_r12 843 -#define _GUARD_NOS_TUPLE_r22 844 -#define _GUARD_NOS_TUPLE_r33 845 -#define _GUARD_NOS_UNICODE_r02 846 -#define _GUARD_NOS_UNICODE_r12 847 -#define _GUARD_NOS_UNICODE_r22 848 -#define _GUARD_NOS_UNICODE_r33 849 -#define _GUARD_NOT_EXHAUSTED_LIST_r02 850 -#define _GUARD_NOT_EXHAUSTED_LIST_r12 851 -#define _GUARD_NOT_EXHAUSTED_LIST_r22 852 -#define _GUARD_NOT_EXHAUSTED_LIST_r33 853 -#define _GUARD_NOT_EXHAUSTED_RANGE_r02 854 -#define _GUARD_NOT_EXHAUSTED_RANGE_r12 855 -#define _GUARD_NOT_EXHAUSTED_RANGE_r22 856 -#define _GUARD_NOT_EXHAUSTED_RANGE_r33 857 -#define _GUARD_NOT_EXHAUSTED_TUPLE_r02 858 -#define _GUARD_NOT_EXHAUSTED_TUPLE_r12 859 -#define _GUARD_NOT_EXHAUSTED_TUPLE_r22 860 -#define _GUARD_NOT_EXHAUSTED_TUPLE_r33 861 -#define _GUARD_THIRD_NULL_r03 862 -#define _GUARD_THIRD_NULL_r13 863 -#define _GUARD_THIRD_NULL_r23 864 -#define _GUARD_THIRD_NULL_r33 865 -#define _GUARD_TOS_ANY_SET_r01 866 -#define _GUARD_TOS_ANY_SET_r11 867 -#define _GUARD_TOS_ANY_SET_r22 868 -#define _GUARD_TOS_ANY_SET_r33 869 -#define _GUARD_TOS_DICT_r01 870 -#define _GUARD_TOS_DICT_r11 871 -#define _GUARD_TOS_DICT_r22 872 -#define _GUARD_TOS_DICT_r33 873 -#define _GUARD_TOS_FLOAT_r01 874 -#define _GUARD_TOS_FLOAT_r11 875 -#define _GUARD_TOS_FLOAT_r22 876 -#define _GUARD_TOS_FLOAT_r33 877 -#define _GUARD_TOS_INT_r01 878 -#define _GUARD_TOS_INT_r11 879 -#define _GUARD_TOS_INT_r22 880 -#define _GUARD_TOS_INT_r33 881 -#define _GUARD_TOS_LIST_r01 882 -#define _GUARD_TOS_LIST_r11 883 -#define _GUARD_TOS_LIST_r22 884 -#define _GUARD_TOS_LIST_r33 885 -#define _GUARD_TOS_OVERFLOWED_r01 886 -#define _GUARD_TOS_OVERFLOWED_r11 887 -#define _GUARD_TOS_OVERFLOWED_r22 888 -#define _GUARD_TOS_OVERFLOWED_r33 889 -#define _GUARD_TOS_SLICE_r01 890 -#define _GUARD_TOS_SLICE_r11 891 -#define _GUARD_TOS_SLICE_r22 892 -#define _GUARD_TOS_SLICE_r33 893 -#define _GUARD_TOS_TUPLE_r01 894 -#define _GUARD_TOS_TUPLE_r11 895 -#define _GUARD_TOS_TUPLE_r22 896 -#define _GUARD_TOS_TUPLE_r33 897 -#define _GUARD_TOS_UNICODE_r01 898 -#define _GUARD_TOS_UNICODE_r11 899 -#define _GUARD_TOS_UNICODE_r22 900 -#define _GUARD_TOS_UNICODE_r33 901 -#define _GUARD_TYPE_VERSION_r01 902 -#define _GUARD_TYPE_VERSION_r11 903 -#define _GUARD_TYPE_VERSION_r22 904 -#define _GUARD_TYPE_VERSION_r33 905 -#define _GUARD_TYPE_VERSION_AND_LOCK_r01 906 -#define _GUARD_TYPE_VERSION_AND_LOCK_r11 907 -#define _GUARD_TYPE_VERSION_AND_LOCK_r22 908 -#define _GUARD_TYPE_VERSION_AND_LOCK_r33 909 -#define _HANDLE_PENDING_AND_DEOPT_r00 910 -#define _HANDLE_PENDING_AND_DEOPT_r10 911 -#define _HANDLE_PENDING_AND_DEOPT_r20 912 -#define _HANDLE_PENDING_AND_DEOPT_r30 913 -#define _IMPORT_FROM_r12 914 -#define _IMPORT_NAME_r21 915 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00 916 -#define _INIT_CALL_PY_EXACT_ARGS_r01 917 -#define _INIT_CALL_PY_EXACT_ARGS_0_r01 918 -#define _INIT_CALL_PY_EXACT_ARGS_1_r01 919 -#define _INIT_CALL_PY_EXACT_ARGS_2_r01 920 -#define _INIT_CALL_PY_EXACT_ARGS_3_r01 921 -#define _INIT_CALL_PY_EXACT_ARGS_4_r01 922 -#define _INSERT_NULL_r10 923 -#define _INSTRUMENTED_FOR_ITER_r23 924 -#define _INSTRUMENTED_INSTRUCTION_r00 925 -#define _INSTRUMENTED_JUMP_FORWARD_r00 926 -#define _INSTRUMENTED_JUMP_FORWARD_r11 927 -#define _INSTRUMENTED_JUMP_FORWARD_r22 928 -#define _INSTRUMENTED_JUMP_FORWARD_r33 929 -#define _INSTRUMENTED_LINE_r00 930 -#define _INSTRUMENTED_NOT_TAKEN_r00 931 -#define _INSTRUMENTED_NOT_TAKEN_r11 932 -#define _INSTRUMENTED_NOT_TAKEN_r22 933 -#define _INSTRUMENTED_NOT_TAKEN_r33 934 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r00 935 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r10 936 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r21 937 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r32 938 -#define _INSTRUMENTED_POP_JUMP_IF_NONE_r10 939 -#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE_r10 940 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r00 941 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r10 942 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r21 943 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r32 944 -#define _IS_NONE_r11 945 -#define _IS_OP_r21 946 -#define _ITER_CHECK_LIST_r02 947 -#define _ITER_CHECK_LIST_r12 948 -#define _ITER_CHECK_LIST_r22 949 -#define _ITER_CHECK_LIST_r33 950 -#define _ITER_CHECK_RANGE_r02 951 -#define _ITER_CHECK_RANGE_r12 952 -#define _ITER_CHECK_RANGE_r22 953 -#define _ITER_CHECK_RANGE_r33 954 -#define _ITER_CHECK_TUPLE_r02 955 -#define _ITER_CHECK_TUPLE_r12 956 -#define _ITER_CHECK_TUPLE_r22 957 -#define _ITER_CHECK_TUPLE_r33 958 -#define _ITER_JUMP_LIST_r02 959 -#define _ITER_JUMP_LIST_r12 960 -#define _ITER_JUMP_LIST_r22 961 -#define _ITER_JUMP_LIST_r33 962 -#define _ITER_JUMP_RANGE_r02 963 -#define _ITER_JUMP_RANGE_r12 964 -#define _ITER_JUMP_RANGE_r22 965 -#define _ITER_JUMP_RANGE_r33 966 -#define _ITER_JUMP_TUPLE_r02 967 -#define _ITER_JUMP_TUPLE_r12 968 -#define _ITER_JUMP_TUPLE_r22 969 -#define _ITER_JUMP_TUPLE_r33 970 -#define _ITER_NEXT_LIST_r23 971 -#define _ITER_NEXT_LIST_TIER_TWO_r23 972 -#define _ITER_NEXT_RANGE_r03 973 -#define _ITER_NEXT_RANGE_r13 974 -#define _ITER_NEXT_RANGE_r23 975 -#define _ITER_NEXT_TUPLE_r03 976 -#define _ITER_NEXT_TUPLE_r13 977 -#define _ITER_NEXT_TUPLE_r23 978 -#define _JUMP_BACKWARD_NO_INTERRUPT_r00 979 -#define _JUMP_BACKWARD_NO_INTERRUPT_r11 980 -#define _JUMP_BACKWARD_NO_INTERRUPT_r22 981 -#define _JUMP_BACKWARD_NO_INTERRUPT_r33 982 -#define _JUMP_TO_TOP_r00 983 -#define _LIST_APPEND_r10 984 -#define _LIST_EXTEND_r10 985 -#define _LOAD_ATTR_r10 986 -#define _LOAD_ATTR_CLASS_r11 987 -#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_r11 988 -#define _LOAD_ATTR_INSTANCE_VALUE_r02 989 -#define _LOAD_ATTR_INSTANCE_VALUE_r12 990 -#define _LOAD_ATTR_INSTANCE_VALUE_r23 991 -#define _LOAD_ATTR_METHOD_LAZY_DICT_r02 992 -#define _LOAD_ATTR_METHOD_LAZY_DICT_r12 993 -#define _LOAD_ATTR_METHOD_LAZY_DICT_r23 994 -#define _LOAD_ATTR_METHOD_NO_DICT_r02 995 -#define _LOAD_ATTR_METHOD_NO_DICT_r12 996 -#define _LOAD_ATTR_METHOD_NO_DICT_r23 997 -#define _LOAD_ATTR_METHOD_WITH_VALUES_r02 998 -#define _LOAD_ATTR_METHOD_WITH_VALUES_r12 999 -#define _LOAD_ATTR_METHOD_WITH_VALUES_r23 1000 -#define _LOAD_ATTR_MODULE_r11 1001 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11 1002 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11 1003 -#define _LOAD_ATTR_PROPERTY_FRAME_r11 1004 -#define _LOAD_ATTR_SLOT_r11 1005 -#define _LOAD_ATTR_WITH_HINT_r12 1006 -#define _LOAD_BUILD_CLASS_r01 1007 -#define _LOAD_BYTECODE_r00 1008 -#define _LOAD_COMMON_CONSTANT_r01 1009 -#define _LOAD_COMMON_CONSTANT_r12 1010 -#define _LOAD_COMMON_CONSTANT_r23 1011 -#define _LOAD_CONST_r01 1012 -#define _LOAD_CONST_r12 1013 -#define _LOAD_CONST_r23 1014 -#define _LOAD_CONST_INLINE_r01 1015 -#define _LOAD_CONST_INLINE_r12 1016 -#define _LOAD_CONST_INLINE_r23 1017 -#define _LOAD_CONST_INLINE_BORROW_r01 1018 -#define _LOAD_CONST_INLINE_BORROW_r12 1019 -#define _LOAD_CONST_INLINE_BORROW_r23 1020 -#define _LOAD_CONST_UNDER_INLINE_r02 1021 -#define _LOAD_CONST_UNDER_INLINE_r12 1022 -#define _LOAD_CONST_UNDER_INLINE_r23 1023 -#define _LOAD_CONST_UNDER_INLINE_BORROW_r02 1024 -#define _LOAD_CONST_UNDER_INLINE_BORROW_r12 1025 -#define _LOAD_CONST_UNDER_INLINE_BORROW_r23 1026 -#define _LOAD_DEREF_r01 1027 -#define _LOAD_FAST_r01 1028 -#define _LOAD_FAST_r12 1029 -#define _LOAD_FAST_r23 1030 -#define _LOAD_FAST_0_r01 1031 -#define _LOAD_FAST_0_r12 1032 -#define _LOAD_FAST_0_r23 1033 -#define _LOAD_FAST_1_r01 1034 -#define _LOAD_FAST_1_r12 1035 -#define _LOAD_FAST_1_r23 1036 -#define _LOAD_FAST_2_r01 1037 -#define _LOAD_FAST_2_r12 1038 -#define _LOAD_FAST_2_r23 1039 -#define _LOAD_FAST_3_r01 1040 -#define _LOAD_FAST_3_r12 1041 -#define _LOAD_FAST_3_r23 1042 -#define _LOAD_FAST_4_r01 1043 -#define _LOAD_FAST_4_r12 1044 -#define _LOAD_FAST_4_r23 1045 -#define _LOAD_FAST_5_r01 1046 -#define _LOAD_FAST_5_r12 1047 -#define _LOAD_FAST_5_r23 1048 -#define _LOAD_FAST_6_r01 1049 -#define _LOAD_FAST_6_r12 1050 -#define _LOAD_FAST_6_r23 1051 -#define _LOAD_FAST_7_r01 1052 -#define _LOAD_FAST_7_r12 1053 -#define _LOAD_FAST_7_r23 1054 -#define _LOAD_FAST_AND_CLEAR_r01 1055 -#define _LOAD_FAST_AND_CLEAR_r12 1056 -#define _LOAD_FAST_AND_CLEAR_r23 1057 -#define _LOAD_FAST_BORROW_r01 1058 -#define _LOAD_FAST_BORROW_r12 1059 -#define _LOAD_FAST_BORROW_r23 1060 -#define _LOAD_FAST_BORROW_0_r01 1061 -#define _LOAD_FAST_BORROW_0_r12 1062 -#define _LOAD_FAST_BORROW_0_r23 1063 -#define _LOAD_FAST_BORROW_1_r01 1064 -#define _LOAD_FAST_BORROW_1_r12 1065 -#define _LOAD_FAST_BORROW_1_r23 1066 -#define _LOAD_FAST_BORROW_2_r01 1067 -#define _LOAD_FAST_BORROW_2_r12 1068 -#define _LOAD_FAST_BORROW_2_r23 1069 -#define _LOAD_FAST_BORROW_3_r01 1070 -#define _LOAD_FAST_BORROW_3_r12 1071 -#define _LOAD_FAST_BORROW_3_r23 1072 -#define _LOAD_FAST_BORROW_4_r01 1073 -#define _LOAD_FAST_BORROW_4_r12 1074 -#define _LOAD_FAST_BORROW_4_r23 1075 -#define _LOAD_FAST_BORROW_5_r01 1076 -#define _LOAD_FAST_BORROW_5_r12 1077 -#define _LOAD_FAST_BORROW_5_r23 1078 -#define _LOAD_FAST_BORROW_6_r01 1079 -#define _LOAD_FAST_BORROW_6_r12 1080 -#define _LOAD_FAST_BORROW_6_r23 1081 -#define _LOAD_FAST_BORROW_7_r01 1082 -#define _LOAD_FAST_BORROW_7_r12 1083 -#define _LOAD_FAST_BORROW_7_r23 1084 -#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r02 1085 -#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r13 1086 -#define _LOAD_FAST_CHECK_r01 1087 -#define _LOAD_FAST_CHECK_r12 1088 -#define _LOAD_FAST_CHECK_r23 1089 -#define _LOAD_FAST_LOAD_FAST_r02 1090 -#define _LOAD_FAST_LOAD_FAST_r13 1091 -#define _LOAD_FROM_DICT_OR_DEREF_r11 1092 -#define _LOAD_FROM_DICT_OR_GLOBALS_r11 1093 -#define _LOAD_GLOBAL_r00 1094 -#define _LOAD_GLOBAL_BUILTINS_r01 1095 -#define _LOAD_GLOBAL_MODULE_r01 1096 -#define _LOAD_LOCALS_r01 1097 -#define _LOAD_LOCALS_r12 1098 -#define _LOAD_LOCALS_r23 1099 -#define _LOAD_NAME_r01 1100 -#define _LOAD_SMALL_INT_r01 1101 -#define _LOAD_SMALL_INT_r12 1102 -#define _LOAD_SMALL_INT_r23 1103 -#define _LOAD_SMALL_INT_0_r01 1104 -#define _LOAD_SMALL_INT_0_r12 1105 -#define _LOAD_SMALL_INT_0_r23 1106 -#define _LOAD_SMALL_INT_1_r01 1107 -#define _LOAD_SMALL_INT_1_r12 1108 -#define _LOAD_SMALL_INT_1_r23 1109 -#define _LOAD_SMALL_INT_2_r01 1110 -#define _LOAD_SMALL_INT_2_r12 1111 -#define _LOAD_SMALL_INT_2_r23 1112 -#define _LOAD_SMALL_INT_3_r01 1113 -#define _LOAD_SMALL_INT_3_r12 1114 -#define _LOAD_SMALL_INT_3_r23 1115 -#define _LOAD_SPECIAL_r00 1116 -#define _LOAD_SUPER_ATTR_ATTR_r31 1117 -#define _LOAD_SUPER_ATTR_METHOD_r32 1118 -#define _MAKE_CALLARGS_A_TUPLE_r33 1119 -#define _MAKE_CELL_r00 1120 -#define _MAKE_FUNCTION_r11 1121 -#define _MAKE_WARM_r00 1122 -#define _MAKE_WARM_r11 1123 -#define _MAKE_WARM_r22 1124 -#define _MAKE_WARM_r33 1125 -#define _MAP_ADD_r20 1126 -#define _MATCH_CLASS_r31 1127 -#define _MATCH_KEYS_r23 1128 -#define _MATCH_MAPPING_r02 1129 -#define _MATCH_MAPPING_r12 1130 -#define _MATCH_MAPPING_r23 1131 -#define _MATCH_SEQUENCE_r02 1132 -#define _MATCH_SEQUENCE_r12 1133 -#define _MATCH_SEQUENCE_r23 1134 -#define _MAYBE_EXPAND_METHOD_r00 1135 -#define _MAYBE_EXPAND_METHOD_KW_r11 1136 -#define _MONITOR_CALL_r00 1137 -#define _MONITOR_CALL_KW_r11 1138 -#define _MONITOR_JUMP_BACKWARD_r00 1139 -#define _MONITOR_JUMP_BACKWARD_r11 1140 -#define _MONITOR_JUMP_BACKWARD_r22 1141 -#define _MONITOR_JUMP_BACKWARD_r33 1142 -#define _MONITOR_RESUME_r00 1143 -#define _NOP_r00 1144 -#define _NOP_r11 1145 -#define _NOP_r22 1146 -#define _NOP_r33 1147 -#define _POP_CALL_r20 1148 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW_r21 1149 -#define _POP_CALL_ONE_r30 1150 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW_r31 1151 -#define _POP_CALL_TWO_r30 1152 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW_r31 1153 -#define _POP_EXCEPT_r10 1154 -#define _POP_ITER_r20 1155 -#define _POP_JUMP_IF_FALSE_r00 1156 -#define _POP_JUMP_IF_FALSE_r10 1157 -#define _POP_JUMP_IF_FALSE_r21 1158 -#define _POP_JUMP_IF_FALSE_r32 1159 -#define _POP_JUMP_IF_TRUE_r00 1160 -#define _POP_JUMP_IF_TRUE_r10 1161 -#define _POP_JUMP_IF_TRUE_r21 1162 -#define _POP_JUMP_IF_TRUE_r32 1163 -#define _POP_TOP_r10 1164 -#define _POP_TOP_FLOAT_r00 1165 -#define _POP_TOP_FLOAT_r10 1166 -#define _POP_TOP_FLOAT_r21 1167 -#define _POP_TOP_FLOAT_r32 1168 -#define _POP_TOP_INT_r00 1169 -#define _POP_TOP_INT_r10 1170 -#define _POP_TOP_INT_r21 1171 -#define _POP_TOP_INT_r32 1172 -#define _POP_TOP_LOAD_CONST_INLINE_r11 1173 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW_r11 1174 -#define _POP_TOP_NOP_r00 1175 -#define _POP_TOP_NOP_r10 1176 -#define _POP_TOP_NOP_r21 1177 -#define _POP_TOP_NOP_r32 1178 -#define _POP_TOP_UNICODE_r00 1179 -#define _POP_TOP_UNICODE_r10 1180 -#define _POP_TOP_UNICODE_r21 1181 -#define _POP_TOP_UNICODE_r32 1182 -#define _POP_TWO_r20 1183 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW_r21 1184 -#define _PUSH_EXC_INFO_r02 1185 -#define _PUSH_EXC_INFO_r12 1186 -#define _PUSH_EXC_INFO_r23 1187 -#define _PUSH_FRAME_r10 1188 -#define _PUSH_NULL_r01 1189 -#define _PUSH_NULL_r12 1190 -#define _PUSH_NULL_r23 1191 -#define _PUSH_NULL_CONDITIONAL_r00 1192 -#define _PY_FRAME_GENERAL_r01 1193 -#define _PY_FRAME_KW_r11 1194 -#define _QUICKEN_RESUME_r00 1195 -#define _QUICKEN_RESUME_r11 1196 -#define _QUICKEN_RESUME_r22 1197 -#define _QUICKEN_RESUME_r33 1198 -#define _REPLACE_WITH_TRUE_r11 1199 -#define _RESUME_CHECK_r00 1200 -#define _RESUME_CHECK_r11 1201 -#define _RESUME_CHECK_r22 1202 -#define _RESUME_CHECK_r33 1203 -#define _RETURN_GENERATOR_r01 1204 -#define _RETURN_VALUE_r11 1205 -#define _SAVE_RETURN_OFFSET_r00 1206 -#define _SAVE_RETURN_OFFSET_r11 1207 -#define _SAVE_RETURN_OFFSET_r22 1208 -#define _SAVE_RETURN_OFFSET_r33 1209 -#define _SEND_r22 1210 -#define _SEND_GEN_FRAME_r22 1211 -#define _SETUP_ANNOTATIONS_r00 1212 -#define _SET_ADD_r10 1213 -#define _SET_FUNCTION_ATTRIBUTE_r01 1214 -#define _SET_FUNCTION_ATTRIBUTE_r11 1215 -#define _SET_FUNCTION_ATTRIBUTE_r21 1216 -#define _SET_FUNCTION_ATTRIBUTE_r32 1217 -#define _SET_IP_r00 1218 -#define _SET_IP_r11 1219 -#define _SET_IP_r22 1220 -#define _SET_IP_r33 1221 -#define _SET_UPDATE_r10 1222 -#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r02 1223 -#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r12 1224 -#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r22 1225 -#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r32 1226 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03 1227 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13 1228 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23 1229 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33 1230 -#define _SPILL_OR_RELOAD_r01 1231 -#define _SPILL_OR_RELOAD_r02 1232 -#define _SPILL_OR_RELOAD_r03 1233 -#define _SPILL_OR_RELOAD_r10 1234 -#define _SPILL_OR_RELOAD_r12 1235 -#define _SPILL_OR_RELOAD_r13 1236 -#define _SPILL_OR_RELOAD_r20 1237 -#define _SPILL_OR_RELOAD_r21 1238 -#define _SPILL_OR_RELOAD_r23 1239 -#define _SPILL_OR_RELOAD_r30 1240 -#define _SPILL_OR_RELOAD_r31 1241 -#define _SPILL_OR_RELOAD_r32 1242 -#define _START_EXECUTOR_r00 1243 -#define _STORE_ATTR_r20 1244 -#define _STORE_ATTR_INSTANCE_VALUE_r21 1245 -#define _STORE_ATTR_SLOT_r21 1246 -#define _STORE_ATTR_WITH_HINT_r21 1247 -#define _STORE_DEREF_r10 1248 -#define _STORE_FAST_r10 1249 -#define _STORE_FAST_0_r10 1250 -#define _STORE_FAST_1_r10 1251 -#define _STORE_FAST_2_r10 1252 -#define _STORE_FAST_3_r10 1253 -#define _STORE_FAST_4_r10 1254 -#define _STORE_FAST_5_r10 1255 -#define _STORE_FAST_6_r10 1256 -#define _STORE_FAST_7_r10 1257 -#define _STORE_FAST_LOAD_FAST_r11 1258 -#define _STORE_FAST_STORE_FAST_r20 1259 -#define _STORE_GLOBAL_r10 1260 -#define _STORE_NAME_r10 1261 -#define _STORE_SLICE_r30 1262 -#define _STORE_SUBSCR_r30 1263 -#define _STORE_SUBSCR_DICT_r31 1264 -#define _STORE_SUBSCR_LIST_INT_r32 1265 -#define _SWAP_r11 1266 -#define _SWAP_2_r02 1267 -#define _SWAP_2_r12 1268 -#define _SWAP_2_r22 1269 -#define _SWAP_2_r33 1270 -#define _SWAP_3_r03 1271 -#define _SWAP_3_r13 1272 -#define _SWAP_3_r23 1273 -#define _SWAP_3_r33 1274 -#define _TIER2_RESUME_CHECK_r00 1275 -#define _TIER2_RESUME_CHECK_r11 1276 -#define _TIER2_RESUME_CHECK_r22 1277 -#define _TIER2_RESUME_CHECK_r33 1278 -#define _TO_BOOL_r11 1279 -#define _TO_BOOL_BOOL_r01 1280 -#define _TO_BOOL_BOOL_r11 1281 -#define _TO_BOOL_BOOL_r22 1282 -#define _TO_BOOL_BOOL_r33 1283 -#define _TO_BOOL_INT_r11 1284 -#define _TO_BOOL_LIST_r11 1285 -#define _TO_BOOL_NONE_r01 1286 -#define _TO_BOOL_NONE_r11 1287 -#define _TO_BOOL_NONE_r22 1288 -#define _TO_BOOL_NONE_r33 1289 -#define _TO_BOOL_STR_r11 1290 -#define _TRACE_RECORD_r00 1291 -#define _UNARY_INVERT_r11 1292 -#define _UNARY_NEGATIVE_r11 1293 -#define _UNARY_NOT_r01 1294 -#define _UNARY_NOT_r11 1295 -#define _UNARY_NOT_r22 1296 -#define _UNARY_NOT_r33 1297 -#define _UNPACK_EX_r10 1298 -#define _UNPACK_SEQUENCE_r10 1299 -#define _UNPACK_SEQUENCE_LIST_r10 1300 -#define _UNPACK_SEQUENCE_TUPLE_r10 1301 -#define _UNPACK_SEQUENCE_TWO_TUPLE_r12 1302 -#define _WITH_EXCEPT_START_r33 1303 -#define _YIELD_VALUE_r11 1304 -#define MAX_UOP_REGS_ID 1304 +#define MAX_UOP_ID 556 +#define _BINARY_OP_r21 557 +#define _BINARY_OP_ADD_FLOAT_r03 558 +#define _BINARY_OP_ADD_FLOAT_r13 559 +#define _BINARY_OP_ADD_FLOAT_r23 560 +#define _BINARY_OP_ADD_INT_r03 561 +#define _BINARY_OP_ADD_INT_r13 562 +#define _BINARY_OP_ADD_INT_r23 563 +#define _BINARY_OP_ADD_UNICODE_r03 564 +#define _BINARY_OP_ADD_UNICODE_r13 565 +#define _BINARY_OP_ADD_UNICODE_r23 566 +#define _BINARY_OP_EXTEND_r21 567 +#define _BINARY_OP_INPLACE_ADD_UNICODE_r21 568 +#define _BINARY_OP_MULTIPLY_FLOAT_r03 569 +#define _BINARY_OP_MULTIPLY_FLOAT_r13 570 +#define _BINARY_OP_MULTIPLY_FLOAT_r23 571 +#define _BINARY_OP_MULTIPLY_INT_r03 572 +#define _BINARY_OP_MULTIPLY_INT_r13 573 +#define _BINARY_OP_MULTIPLY_INT_r23 574 +#define _BINARY_OP_SUBSCR_CHECK_FUNC_r23 575 +#define _BINARY_OP_SUBSCR_DICT_r21 576 +#define _BINARY_OP_SUBSCR_INIT_CALL_r01 577 +#define _BINARY_OP_SUBSCR_INIT_CALL_r11 578 +#define _BINARY_OP_SUBSCR_INIT_CALL_r21 579 +#define _BINARY_OP_SUBSCR_INIT_CALL_r31 580 +#define _BINARY_OP_SUBSCR_LIST_INT_r23 581 +#define _BINARY_OP_SUBSCR_LIST_SLICE_r21 582 +#define _BINARY_OP_SUBSCR_STR_INT_r23 583 +#define _BINARY_OP_SUBSCR_TUPLE_INT_r23 584 +#define _BINARY_OP_SUBTRACT_FLOAT_r03 585 +#define _BINARY_OP_SUBTRACT_FLOAT_r13 586 +#define _BINARY_OP_SUBTRACT_FLOAT_r23 587 +#define _BINARY_OP_SUBTRACT_INT_r03 588 +#define _BINARY_OP_SUBTRACT_INT_r13 589 +#define _BINARY_OP_SUBTRACT_INT_r23 590 +#define _BINARY_SLICE_r31 591 +#define _BUILD_INTERPOLATION_r01 592 +#define _BUILD_LIST_r01 593 +#define _BUILD_MAP_r01 594 +#define _BUILD_SET_r01 595 +#define _BUILD_SLICE_r01 596 +#define _BUILD_STRING_r01 597 +#define _BUILD_TEMPLATE_r21 598 +#define _BUILD_TUPLE_r01 599 +#define _CALL_BUILTIN_CLASS_r01 600 +#define _CALL_BUILTIN_FAST_r01 601 +#define _CALL_BUILTIN_FAST_WITH_KEYWORDS_r01 602 +#define _CALL_BUILTIN_O_r03 603 +#define _CALL_INTRINSIC_1_r11 604 +#define _CALL_INTRINSIC_2_r21 605 +#define _CALL_ISINSTANCE_r31 606 +#define _CALL_KW_NON_PY_r11 607 +#define _CALL_LEN_r33 608 +#define _CALL_LIST_APPEND_r03 609 +#define _CALL_LIST_APPEND_r13 610 +#define _CALL_LIST_APPEND_r23 611 +#define _CALL_LIST_APPEND_r33 612 +#define _CALL_METHOD_DESCRIPTOR_FAST_r01 613 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r01 614 +#define _CALL_METHOD_DESCRIPTOR_NOARGS_r01 615 +#define _CALL_METHOD_DESCRIPTOR_O_r01 616 +#define _CALL_NON_PY_GENERAL_r01 617 +#define _CALL_STR_1_r32 618 +#define _CALL_TUPLE_1_r32 619 +#define _CALL_TYPE_1_r02 620 +#define _CALL_TYPE_1_r12 621 +#define _CALL_TYPE_1_r22 622 +#define _CALL_TYPE_1_r32 623 +#define _CHECK_AND_ALLOCATE_OBJECT_r00 624 +#define _CHECK_ATTR_CLASS_r01 625 +#define _CHECK_ATTR_CLASS_r11 626 +#define _CHECK_ATTR_CLASS_r22 627 +#define _CHECK_ATTR_CLASS_r33 628 +#define _CHECK_ATTR_METHOD_LAZY_DICT_r01 629 +#define _CHECK_ATTR_METHOD_LAZY_DICT_r11 630 +#define _CHECK_ATTR_METHOD_LAZY_DICT_r22 631 +#define _CHECK_ATTR_METHOD_LAZY_DICT_r33 632 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00 633 +#define _CHECK_EG_MATCH_r22 634 +#define _CHECK_EXC_MATCH_r22 635 +#define _CHECK_FUNCTION_EXACT_ARGS_r00 636 +#define _CHECK_FUNCTION_VERSION_r00 637 +#define _CHECK_FUNCTION_VERSION_INLINE_r00 638 +#define _CHECK_FUNCTION_VERSION_INLINE_r11 639 +#define _CHECK_FUNCTION_VERSION_INLINE_r22 640 +#define _CHECK_FUNCTION_VERSION_INLINE_r33 641 +#define _CHECK_FUNCTION_VERSION_KW_r11 642 +#define _CHECK_IS_NOT_PY_CALLABLE_r00 643 +#define _CHECK_IS_NOT_PY_CALLABLE_KW_r11 644 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r01 645 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r11 646 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r22 647 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r33 648 +#define _CHECK_METHOD_VERSION_r00 649 +#define _CHECK_METHOD_VERSION_KW_r11 650 +#define _CHECK_PEP_523_r00 651 +#define _CHECK_PEP_523_r11 652 +#define _CHECK_PEP_523_r22 653 +#define _CHECK_PEP_523_r33 654 +#define _CHECK_PERIODIC_r00 655 +#define _CHECK_PERIODIC_AT_END_r00 656 +#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00 657 +#define _CHECK_RECURSION_REMAINING_r00 658 +#define _CHECK_RECURSION_REMAINING_r11 659 +#define _CHECK_RECURSION_REMAINING_r22 660 +#define _CHECK_RECURSION_REMAINING_r33 661 +#define _CHECK_STACK_SPACE_r00 662 +#define _CHECK_STACK_SPACE_OPERAND_r00 663 +#define _CHECK_STACK_SPACE_OPERAND_r11 664 +#define _CHECK_STACK_SPACE_OPERAND_r22 665 +#define _CHECK_STACK_SPACE_OPERAND_r33 666 +#define _CHECK_VALIDITY_r00 667 +#define _CHECK_VALIDITY_r11 668 +#define _CHECK_VALIDITY_r22 669 +#define _CHECK_VALIDITY_r33 670 +#define _COLD_DYNAMIC_EXIT_r00 671 +#define _COLD_EXIT_r00 672 +#define _COMPARE_OP_r21 673 +#define _COMPARE_OP_FLOAT_r03 674 +#define _COMPARE_OP_FLOAT_r13 675 +#define _COMPARE_OP_FLOAT_r23 676 +#define _COMPARE_OP_INT_r23 677 +#define _COMPARE_OP_STR_r23 678 +#define _CONTAINS_OP_r21 679 +#define _CONTAINS_OP_DICT_r21 680 +#define _CONTAINS_OP_SET_r21 681 +#define _CONVERT_VALUE_r11 682 +#define _COPY_r01 683 +#define _COPY_1_r02 684 +#define _COPY_1_r12 685 +#define _COPY_1_r23 686 +#define _COPY_2_r03 687 +#define _COPY_2_r13 688 +#define _COPY_2_r23 689 +#define _COPY_3_r03 690 +#define _COPY_3_r13 691 +#define _COPY_3_r23 692 +#define _COPY_3_r33 693 +#define _COPY_FREE_VARS_r00 694 +#define _COPY_FREE_VARS_r11 695 +#define _COPY_FREE_VARS_r22 696 +#define _COPY_FREE_VARS_r33 697 +#define _CREATE_INIT_FRAME_r01 698 +#define _DELETE_ATTR_r10 699 +#define _DELETE_DEREF_r00 700 +#define _DELETE_FAST_r00 701 +#define _DELETE_GLOBAL_r00 702 +#define _DELETE_NAME_r00 703 +#define _DELETE_SUBSCR_r20 704 +#define _DEOPT_r00 705 +#define _DEOPT_r10 706 +#define _DEOPT_r20 707 +#define _DEOPT_r30 708 +#define _DICT_MERGE_r10 709 +#define _DICT_UPDATE_r10 710 +#define _DO_CALL_r01 711 +#define _DO_CALL_FUNCTION_EX_r31 712 +#define _DO_CALL_KW_r11 713 +#define _DYNAMIC_EXIT_r00 714 +#define _DYNAMIC_EXIT_r10 715 +#define _DYNAMIC_EXIT_r20 716 +#define _DYNAMIC_EXIT_r30 717 +#define _END_FOR_r10 718 +#define _END_SEND_r21 719 +#define _ERROR_POP_N_r00 720 +#define _EXIT_INIT_CHECK_r10 721 +#define _EXIT_TRACE_r00 722 +#define _EXIT_TRACE_r10 723 +#define _EXIT_TRACE_r20 724 +#define _EXIT_TRACE_r30 725 +#define _EXPAND_METHOD_r00 726 +#define _EXPAND_METHOD_KW_r11 727 +#define _FATAL_ERROR_r00 728 +#define _FATAL_ERROR_r11 729 +#define _FATAL_ERROR_r22 730 +#define _FATAL_ERROR_r33 731 +#define _FORMAT_SIMPLE_r11 732 +#define _FORMAT_WITH_SPEC_r21 733 +#define _FOR_ITER_r23 734 +#define _FOR_ITER_GEN_FRAME_r03 735 +#define _FOR_ITER_GEN_FRAME_r13 736 +#define _FOR_ITER_GEN_FRAME_r23 737 +#define _FOR_ITER_TIER_TWO_r23 738 +#define _GET_AITER_r11 739 +#define _GET_ANEXT_r12 740 +#define _GET_AWAITABLE_r11 741 +#define _GET_ITER_r12 742 +#define _GET_LEN_r12 743 +#define _GET_YIELD_FROM_ITER_r11 744 +#define _GUARD_BINARY_OP_EXTEND_r22 745 +#define _GUARD_CALLABLE_ISINSTANCE_r03 746 +#define _GUARD_CALLABLE_ISINSTANCE_r13 747 +#define _GUARD_CALLABLE_ISINSTANCE_r23 748 +#define _GUARD_CALLABLE_ISINSTANCE_r33 749 +#define _GUARD_CALLABLE_LEN_r03 750 +#define _GUARD_CALLABLE_LEN_r13 751 +#define _GUARD_CALLABLE_LEN_r23 752 +#define _GUARD_CALLABLE_LEN_r33 753 +#define _GUARD_CALLABLE_LIST_APPEND_r03 754 +#define _GUARD_CALLABLE_LIST_APPEND_r13 755 +#define _GUARD_CALLABLE_LIST_APPEND_r23 756 +#define _GUARD_CALLABLE_LIST_APPEND_r33 757 +#define _GUARD_CALLABLE_STR_1_r03 758 +#define _GUARD_CALLABLE_STR_1_r13 759 +#define _GUARD_CALLABLE_STR_1_r23 760 +#define _GUARD_CALLABLE_STR_1_r33 761 +#define _GUARD_CALLABLE_TUPLE_1_r03 762 +#define _GUARD_CALLABLE_TUPLE_1_r13 763 +#define _GUARD_CALLABLE_TUPLE_1_r23 764 +#define _GUARD_CALLABLE_TUPLE_1_r33 765 +#define _GUARD_CALLABLE_TYPE_1_r03 766 +#define _GUARD_CALLABLE_TYPE_1_r13 767 +#define _GUARD_CALLABLE_TYPE_1_r23 768 +#define _GUARD_CALLABLE_TYPE_1_r33 769 +#define _GUARD_DORV_NO_DICT_r01 770 +#define _GUARD_DORV_NO_DICT_r11 771 +#define _GUARD_DORV_NO_DICT_r22 772 +#define _GUARD_DORV_NO_DICT_r33 773 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r01 774 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11 775 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22 776 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33 777 +#define _GUARD_GLOBALS_VERSION_r00 778 +#define _GUARD_GLOBALS_VERSION_r11 779 +#define _GUARD_GLOBALS_VERSION_r22 780 +#define _GUARD_GLOBALS_VERSION_r33 781 +#define _GUARD_IP_RETURN_GENERATOR_r00 782 +#define _GUARD_IP_RETURN_GENERATOR_r11 783 +#define _GUARD_IP_RETURN_GENERATOR_r22 784 +#define _GUARD_IP_RETURN_GENERATOR_r33 785 +#define _GUARD_IP_RETURN_VALUE_r00 786 +#define _GUARD_IP_RETURN_VALUE_r11 787 +#define _GUARD_IP_RETURN_VALUE_r22 788 +#define _GUARD_IP_RETURN_VALUE_r33 789 +#define _GUARD_IP_YIELD_VALUE_r00 790 +#define _GUARD_IP_YIELD_VALUE_r11 791 +#define _GUARD_IP_YIELD_VALUE_r22 792 +#define _GUARD_IP_YIELD_VALUE_r33 793 +#define _GUARD_IP__PUSH_FRAME_r00 794 +#define _GUARD_IP__PUSH_FRAME_r11 795 +#define _GUARD_IP__PUSH_FRAME_r22 796 +#define _GUARD_IP__PUSH_FRAME_r33 797 +#define _GUARD_IS_FALSE_POP_r00 798 +#define _GUARD_IS_FALSE_POP_r10 799 +#define _GUARD_IS_FALSE_POP_r21 800 +#define _GUARD_IS_FALSE_POP_r32 801 +#define _GUARD_IS_NONE_POP_r00 802 +#define _GUARD_IS_NONE_POP_r10 803 +#define _GUARD_IS_NONE_POP_r21 804 +#define _GUARD_IS_NONE_POP_r32 805 +#define _GUARD_IS_NOT_NONE_POP_r10 806 +#define _GUARD_IS_TRUE_POP_r00 807 +#define _GUARD_IS_TRUE_POP_r10 808 +#define _GUARD_IS_TRUE_POP_r21 809 +#define _GUARD_IS_TRUE_POP_r32 810 +#define _GUARD_KEYS_VERSION_r01 811 +#define _GUARD_KEYS_VERSION_r11 812 +#define _GUARD_KEYS_VERSION_r22 813 +#define _GUARD_KEYS_VERSION_r33 814 +#define _GUARD_NOS_DICT_r02 815 +#define _GUARD_NOS_DICT_r12 816 +#define _GUARD_NOS_DICT_r22 817 +#define _GUARD_NOS_DICT_r33 818 +#define _GUARD_NOS_FLOAT_r02 819 +#define _GUARD_NOS_FLOAT_r12 820 +#define _GUARD_NOS_FLOAT_r22 821 +#define _GUARD_NOS_FLOAT_r33 822 +#define _GUARD_NOS_INT_r02 823 +#define _GUARD_NOS_INT_r12 824 +#define _GUARD_NOS_INT_r22 825 +#define _GUARD_NOS_INT_r33 826 +#define _GUARD_NOS_LIST_r02 827 +#define _GUARD_NOS_LIST_r12 828 +#define _GUARD_NOS_LIST_r22 829 +#define _GUARD_NOS_LIST_r33 830 +#define _GUARD_NOS_NOT_NULL_r02 831 +#define _GUARD_NOS_NOT_NULL_r12 832 +#define _GUARD_NOS_NOT_NULL_r22 833 +#define _GUARD_NOS_NOT_NULL_r33 834 +#define _GUARD_NOS_NULL_r02 835 +#define _GUARD_NOS_NULL_r12 836 +#define _GUARD_NOS_NULL_r22 837 +#define _GUARD_NOS_NULL_r33 838 +#define _GUARD_NOS_OVERFLOWED_r02 839 +#define _GUARD_NOS_OVERFLOWED_r12 840 +#define _GUARD_NOS_OVERFLOWED_r22 841 +#define _GUARD_NOS_OVERFLOWED_r33 842 +#define _GUARD_NOS_TUPLE_r02 843 +#define _GUARD_NOS_TUPLE_r12 844 +#define _GUARD_NOS_TUPLE_r22 845 +#define _GUARD_NOS_TUPLE_r33 846 +#define _GUARD_NOS_UNICODE_r02 847 +#define _GUARD_NOS_UNICODE_r12 848 +#define _GUARD_NOS_UNICODE_r22 849 +#define _GUARD_NOS_UNICODE_r33 850 +#define _GUARD_NOT_EXHAUSTED_LIST_r02 851 +#define _GUARD_NOT_EXHAUSTED_LIST_r12 852 +#define _GUARD_NOT_EXHAUSTED_LIST_r22 853 +#define _GUARD_NOT_EXHAUSTED_LIST_r33 854 +#define _GUARD_NOT_EXHAUSTED_RANGE_r02 855 +#define _GUARD_NOT_EXHAUSTED_RANGE_r12 856 +#define _GUARD_NOT_EXHAUSTED_RANGE_r22 857 +#define _GUARD_NOT_EXHAUSTED_RANGE_r33 858 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r02 859 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r12 860 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r22 861 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r33 862 +#define _GUARD_THIRD_NULL_r03 863 +#define _GUARD_THIRD_NULL_r13 864 +#define _GUARD_THIRD_NULL_r23 865 +#define _GUARD_THIRD_NULL_r33 866 +#define _GUARD_TOS_ANY_SET_r01 867 +#define _GUARD_TOS_ANY_SET_r11 868 +#define _GUARD_TOS_ANY_SET_r22 869 +#define _GUARD_TOS_ANY_SET_r33 870 +#define _GUARD_TOS_DICT_r01 871 +#define _GUARD_TOS_DICT_r11 872 +#define _GUARD_TOS_DICT_r22 873 +#define _GUARD_TOS_DICT_r33 874 +#define _GUARD_TOS_FLOAT_r01 875 +#define _GUARD_TOS_FLOAT_r11 876 +#define _GUARD_TOS_FLOAT_r22 877 +#define _GUARD_TOS_FLOAT_r33 878 +#define _GUARD_TOS_INT_r01 879 +#define _GUARD_TOS_INT_r11 880 +#define _GUARD_TOS_INT_r22 881 +#define _GUARD_TOS_INT_r33 882 +#define _GUARD_TOS_LIST_r01 883 +#define _GUARD_TOS_LIST_r11 884 +#define _GUARD_TOS_LIST_r22 885 +#define _GUARD_TOS_LIST_r33 886 +#define _GUARD_TOS_OVERFLOWED_r01 887 +#define _GUARD_TOS_OVERFLOWED_r11 888 +#define _GUARD_TOS_OVERFLOWED_r22 889 +#define _GUARD_TOS_OVERFLOWED_r33 890 +#define _GUARD_TOS_SLICE_r01 891 +#define _GUARD_TOS_SLICE_r11 892 +#define _GUARD_TOS_SLICE_r22 893 +#define _GUARD_TOS_SLICE_r33 894 +#define _GUARD_TOS_TUPLE_r01 895 +#define _GUARD_TOS_TUPLE_r11 896 +#define _GUARD_TOS_TUPLE_r22 897 +#define _GUARD_TOS_TUPLE_r33 898 +#define _GUARD_TOS_UNICODE_r01 899 +#define _GUARD_TOS_UNICODE_r11 900 +#define _GUARD_TOS_UNICODE_r22 901 +#define _GUARD_TOS_UNICODE_r33 902 +#define _GUARD_TYPE_VERSION_r01 903 +#define _GUARD_TYPE_VERSION_r11 904 +#define _GUARD_TYPE_VERSION_r22 905 +#define _GUARD_TYPE_VERSION_r33 906 +#define _GUARD_TYPE_VERSION_AND_LOCK_r01 907 +#define _GUARD_TYPE_VERSION_AND_LOCK_r11 908 +#define _GUARD_TYPE_VERSION_AND_LOCK_r22 909 +#define _GUARD_TYPE_VERSION_AND_LOCK_r33 910 +#define _HANDLE_PENDING_AND_DEOPT_r00 911 +#define _HANDLE_PENDING_AND_DEOPT_r10 912 +#define _HANDLE_PENDING_AND_DEOPT_r20 913 +#define _HANDLE_PENDING_AND_DEOPT_r30 914 +#define _IMPORT_FROM_r12 915 +#define _IMPORT_NAME_r21 916 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00 917 +#define _INIT_CALL_PY_EXACT_ARGS_r01 918 +#define _INIT_CALL_PY_EXACT_ARGS_0_r01 919 +#define _INIT_CALL_PY_EXACT_ARGS_1_r01 920 +#define _INIT_CALL_PY_EXACT_ARGS_2_r01 921 +#define _INIT_CALL_PY_EXACT_ARGS_3_r01 922 +#define _INIT_CALL_PY_EXACT_ARGS_4_r01 923 +#define _INSERT_NULL_r10 924 +#define _INSTRUMENTED_FOR_ITER_r23 925 +#define _INSTRUMENTED_INSTRUCTION_r00 926 +#define _INSTRUMENTED_JUMP_FORWARD_r00 927 +#define _INSTRUMENTED_JUMP_FORWARD_r11 928 +#define _INSTRUMENTED_JUMP_FORWARD_r22 929 +#define _INSTRUMENTED_JUMP_FORWARD_r33 930 +#define _INSTRUMENTED_LINE_r00 931 +#define _INSTRUMENTED_NOT_TAKEN_r00 932 +#define _INSTRUMENTED_NOT_TAKEN_r11 933 +#define _INSTRUMENTED_NOT_TAKEN_r22 934 +#define _INSTRUMENTED_NOT_TAKEN_r33 935 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r00 936 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r10 937 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r21 938 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r32 939 +#define _INSTRUMENTED_POP_JUMP_IF_NONE_r10 940 +#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE_r10 941 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r00 942 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r10 943 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r21 944 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r32 945 +#define _IS_NONE_r11 946 +#define _IS_OP_r03 947 +#define _IS_OP_r13 948 +#define _IS_OP_r23 949 +#define _ITER_CHECK_LIST_r02 950 +#define _ITER_CHECK_LIST_r12 951 +#define _ITER_CHECK_LIST_r22 952 +#define _ITER_CHECK_LIST_r33 953 +#define _ITER_CHECK_RANGE_r02 954 +#define _ITER_CHECK_RANGE_r12 955 +#define _ITER_CHECK_RANGE_r22 956 +#define _ITER_CHECK_RANGE_r33 957 +#define _ITER_CHECK_TUPLE_r02 958 +#define _ITER_CHECK_TUPLE_r12 959 +#define _ITER_CHECK_TUPLE_r22 960 +#define _ITER_CHECK_TUPLE_r33 961 +#define _ITER_JUMP_LIST_r02 962 +#define _ITER_JUMP_LIST_r12 963 +#define _ITER_JUMP_LIST_r22 964 +#define _ITER_JUMP_LIST_r33 965 +#define _ITER_JUMP_RANGE_r02 966 +#define _ITER_JUMP_RANGE_r12 967 +#define _ITER_JUMP_RANGE_r22 968 +#define _ITER_JUMP_RANGE_r33 969 +#define _ITER_JUMP_TUPLE_r02 970 +#define _ITER_JUMP_TUPLE_r12 971 +#define _ITER_JUMP_TUPLE_r22 972 +#define _ITER_JUMP_TUPLE_r33 973 +#define _ITER_NEXT_LIST_r23 974 +#define _ITER_NEXT_LIST_TIER_TWO_r23 975 +#define _ITER_NEXT_RANGE_r03 976 +#define _ITER_NEXT_RANGE_r13 977 +#define _ITER_NEXT_RANGE_r23 978 +#define _ITER_NEXT_TUPLE_r03 979 +#define _ITER_NEXT_TUPLE_r13 980 +#define _ITER_NEXT_TUPLE_r23 981 +#define _JUMP_BACKWARD_NO_INTERRUPT_r00 982 +#define _JUMP_BACKWARD_NO_INTERRUPT_r11 983 +#define _JUMP_BACKWARD_NO_INTERRUPT_r22 984 +#define _JUMP_BACKWARD_NO_INTERRUPT_r33 985 +#define _JUMP_TO_TOP_r00 986 +#define _LIST_APPEND_r10 987 +#define _LIST_EXTEND_r10 988 +#define _LOAD_ATTR_r10 989 +#define _LOAD_ATTR_CLASS_r11 990 +#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_r11 991 +#define _LOAD_ATTR_INSTANCE_VALUE_r02 992 +#define _LOAD_ATTR_INSTANCE_VALUE_r12 993 +#define _LOAD_ATTR_INSTANCE_VALUE_r23 994 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r02 995 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r12 996 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r23 997 +#define _LOAD_ATTR_METHOD_NO_DICT_r02 998 +#define _LOAD_ATTR_METHOD_NO_DICT_r12 999 +#define _LOAD_ATTR_METHOD_NO_DICT_r23 1000 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r02 1001 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r12 1002 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r23 1003 +#define _LOAD_ATTR_MODULE_r11 1004 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11 1005 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11 1006 +#define _LOAD_ATTR_PROPERTY_FRAME_r11 1007 +#define _LOAD_ATTR_SLOT_r11 1008 +#define _LOAD_ATTR_WITH_HINT_r12 1009 +#define _LOAD_BUILD_CLASS_r01 1010 +#define _LOAD_BYTECODE_r00 1011 +#define _LOAD_COMMON_CONSTANT_r01 1012 +#define _LOAD_COMMON_CONSTANT_r12 1013 +#define _LOAD_COMMON_CONSTANT_r23 1014 +#define _LOAD_CONST_r01 1015 +#define _LOAD_CONST_r12 1016 +#define _LOAD_CONST_r23 1017 +#define _LOAD_CONST_INLINE_r01 1018 +#define _LOAD_CONST_INLINE_r12 1019 +#define _LOAD_CONST_INLINE_r23 1020 +#define _LOAD_CONST_INLINE_BORROW_r01 1021 +#define _LOAD_CONST_INLINE_BORROW_r12 1022 +#define _LOAD_CONST_INLINE_BORROW_r23 1023 +#define _LOAD_CONST_UNDER_INLINE_r02 1024 +#define _LOAD_CONST_UNDER_INLINE_r12 1025 +#define _LOAD_CONST_UNDER_INLINE_r23 1026 +#define _LOAD_CONST_UNDER_INLINE_BORROW_r02 1027 +#define _LOAD_CONST_UNDER_INLINE_BORROW_r12 1028 +#define _LOAD_CONST_UNDER_INLINE_BORROW_r23 1029 +#define _LOAD_DEREF_r01 1030 +#define _LOAD_FAST_r01 1031 +#define _LOAD_FAST_r12 1032 +#define _LOAD_FAST_r23 1033 +#define _LOAD_FAST_0_r01 1034 +#define _LOAD_FAST_0_r12 1035 +#define _LOAD_FAST_0_r23 1036 +#define _LOAD_FAST_1_r01 1037 +#define _LOAD_FAST_1_r12 1038 +#define _LOAD_FAST_1_r23 1039 +#define _LOAD_FAST_2_r01 1040 +#define _LOAD_FAST_2_r12 1041 +#define _LOAD_FAST_2_r23 1042 +#define _LOAD_FAST_3_r01 1043 +#define _LOAD_FAST_3_r12 1044 +#define _LOAD_FAST_3_r23 1045 +#define _LOAD_FAST_4_r01 1046 +#define _LOAD_FAST_4_r12 1047 +#define _LOAD_FAST_4_r23 1048 +#define _LOAD_FAST_5_r01 1049 +#define _LOAD_FAST_5_r12 1050 +#define _LOAD_FAST_5_r23 1051 +#define _LOAD_FAST_6_r01 1052 +#define _LOAD_FAST_6_r12 1053 +#define _LOAD_FAST_6_r23 1054 +#define _LOAD_FAST_7_r01 1055 +#define _LOAD_FAST_7_r12 1056 +#define _LOAD_FAST_7_r23 1057 +#define _LOAD_FAST_AND_CLEAR_r01 1058 +#define _LOAD_FAST_AND_CLEAR_r12 1059 +#define _LOAD_FAST_AND_CLEAR_r23 1060 +#define _LOAD_FAST_BORROW_r01 1061 +#define _LOAD_FAST_BORROW_r12 1062 +#define _LOAD_FAST_BORROW_r23 1063 +#define _LOAD_FAST_BORROW_0_r01 1064 +#define _LOAD_FAST_BORROW_0_r12 1065 +#define _LOAD_FAST_BORROW_0_r23 1066 +#define _LOAD_FAST_BORROW_1_r01 1067 +#define _LOAD_FAST_BORROW_1_r12 1068 +#define _LOAD_FAST_BORROW_1_r23 1069 +#define _LOAD_FAST_BORROW_2_r01 1070 +#define _LOAD_FAST_BORROW_2_r12 1071 +#define _LOAD_FAST_BORROW_2_r23 1072 +#define _LOAD_FAST_BORROW_3_r01 1073 +#define _LOAD_FAST_BORROW_3_r12 1074 +#define _LOAD_FAST_BORROW_3_r23 1075 +#define _LOAD_FAST_BORROW_4_r01 1076 +#define _LOAD_FAST_BORROW_4_r12 1077 +#define _LOAD_FAST_BORROW_4_r23 1078 +#define _LOAD_FAST_BORROW_5_r01 1079 +#define _LOAD_FAST_BORROW_5_r12 1080 +#define _LOAD_FAST_BORROW_5_r23 1081 +#define _LOAD_FAST_BORROW_6_r01 1082 +#define _LOAD_FAST_BORROW_6_r12 1083 +#define _LOAD_FAST_BORROW_6_r23 1084 +#define _LOAD_FAST_BORROW_7_r01 1085 +#define _LOAD_FAST_BORROW_7_r12 1086 +#define _LOAD_FAST_BORROW_7_r23 1087 +#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r02 1088 +#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r13 1089 +#define _LOAD_FAST_CHECK_r01 1090 +#define _LOAD_FAST_CHECK_r12 1091 +#define _LOAD_FAST_CHECK_r23 1092 +#define _LOAD_FAST_LOAD_FAST_r02 1093 +#define _LOAD_FAST_LOAD_FAST_r13 1094 +#define _LOAD_FROM_DICT_OR_DEREF_r11 1095 +#define _LOAD_FROM_DICT_OR_GLOBALS_r11 1096 +#define _LOAD_GLOBAL_r00 1097 +#define _LOAD_GLOBAL_BUILTINS_r01 1098 +#define _LOAD_GLOBAL_MODULE_r01 1099 +#define _LOAD_LOCALS_r01 1100 +#define _LOAD_LOCALS_r12 1101 +#define _LOAD_LOCALS_r23 1102 +#define _LOAD_NAME_r01 1103 +#define _LOAD_SMALL_INT_r01 1104 +#define _LOAD_SMALL_INT_r12 1105 +#define _LOAD_SMALL_INT_r23 1106 +#define _LOAD_SMALL_INT_0_r01 1107 +#define _LOAD_SMALL_INT_0_r12 1108 +#define _LOAD_SMALL_INT_0_r23 1109 +#define _LOAD_SMALL_INT_1_r01 1110 +#define _LOAD_SMALL_INT_1_r12 1111 +#define _LOAD_SMALL_INT_1_r23 1112 +#define _LOAD_SMALL_INT_2_r01 1113 +#define _LOAD_SMALL_INT_2_r12 1114 +#define _LOAD_SMALL_INT_2_r23 1115 +#define _LOAD_SMALL_INT_3_r01 1116 +#define _LOAD_SMALL_INT_3_r12 1117 +#define _LOAD_SMALL_INT_3_r23 1118 +#define _LOAD_SPECIAL_r00 1119 +#define _LOAD_SUPER_ATTR_ATTR_r31 1120 +#define _LOAD_SUPER_ATTR_METHOD_r32 1121 +#define _MAKE_CALLARGS_A_TUPLE_r33 1122 +#define _MAKE_CELL_r00 1123 +#define _MAKE_FUNCTION_r11 1124 +#define _MAKE_WARM_r00 1125 +#define _MAKE_WARM_r11 1126 +#define _MAKE_WARM_r22 1127 +#define _MAKE_WARM_r33 1128 +#define _MAP_ADD_r20 1129 +#define _MATCH_CLASS_r31 1130 +#define _MATCH_KEYS_r23 1131 +#define _MATCH_MAPPING_r02 1132 +#define _MATCH_MAPPING_r12 1133 +#define _MATCH_MAPPING_r23 1134 +#define _MATCH_SEQUENCE_r02 1135 +#define _MATCH_SEQUENCE_r12 1136 +#define _MATCH_SEQUENCE_r23 1137 +#define _MAYBE_EXPAND_METHOD_r00 1138 +#define _MAYBE_EXPAND_METHOD_KW_r11 1139 +#define _MONITOR_CALL_r00 1140 +#define _MONITOR_CALL_KW_r11 1141 +#define _MONITOR_JUMP_BACKWARD_r00 1142 +#define _MONITOR_JUMP_BACKWARD_r11 1143 +#define _MONITOR_JUMP_BACKWARD_r22 1144 +#define _MONITOR_JUMP_BACKWARD_r33 1145 +#define _MONITOR_RESUME_r00 1146 +#define _NOP_r00 1147 +#define _NOP_r11 1148 +#define _NOP_r22 1149 +#define _NOP_r33 1150 +#define _POP_CALL_r20 1151 +#define _POP_CALL_LOAD_CONST_INLINE_BORROW_r21 1152 +#define _POP_CALL_ONE_r30 1153 +#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW_r31 1154 +#define _POP_CALL_TWO_r30 1155 +#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW_r31 1156 +#define _POP_EXCEPT_r10 1157 +#define _POP_ITER_r20 1158 +#define _POP_JUMP_IF_FALSE_r00 1159 +#define _POP_JUMP_IF_FALSE_r10 1160 +#define _POP_JUMP_IF_FALSE_r21 1161 +#define _POP_JUMP_IF_FALSE_r32 1162 +#define _POP_JUMP_IF_TRUE_r00 1163 +#define _POP_JUMP_IF_TRUE_r10 1164 +#define _POP_JUMP_IF_TRUE_r21 1165 +#define _POP_JUMP_IF_TRUE_r32 1166 +#define _POP_TOP_r10 1167 +#define _POP_TOP_FLOAT_r00 1168 +#define _POP_TOP_FLOAT_r10 1169 +#define _POP_TOP_FLOAT_r21 1170 +#define _POP_TOP_FLOAT_r32 1171 +#define _POP_TOP_INT_r00 1172 +#define _POP_TOP_INT_r10 1173 +#define _POP_TOP_INT_r21 1174 +#define _POP_TOP_INT_r32 1175 +#define _POP_TOP_LOAD_CONST_INLINE_r11 1176 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW_r11 1177 +#define _POP_TOP_NOP_r00 1178 +#define _POP_TOP_NOP_r10 1179 +#define _POP_TOP_NOP_r21 1180 +#define _POP_TOP_NOP_r32 1181 +#define _POP_TOP_UNICODE_r00 1182 +#define _POP_TOP_UNICODE_r10 1183 +#define _POP_TOP_UNICODE_r21 1184 +#define _POP_TOP_UNICODE_r32 1185 +#define _POP_TWO_r20 1186 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW_r21 1187 +#define _PUSH_EXC_INFO_r02 1188 +#define _PUSH_EXC_INFO_r12 1189 +#define _PUSH_EXC_INFO_r23 1190 +#define _PUSH_FRAME_r10 1191 +#define _PUSH_NULL_r01 1192 +#define _PUSH_NULL_r12 1193 +#define _PUSH_NULL_r23 1194 +#define _PUSH_NULL_CONDITIONAL_r00 1195 +#define _PY_FRAME_GENERAL_r01 1196 +#define _PY_FRAME_KW_r11 1197 +#define _QUICKEN_RESUME_r00 1198 +#define _QUICKEN_RESUME_r11 1199 +#define _QUICKEN_RESUME_r22 1200 +#define _QUICKEN_RESUME_r33 1201 +#define _REPLACE_WITH_TRUE_r11 1202 +#define _RESUME_CHECK_r00 1203 +#define _RESUME_CHECK_r11 1204 +#define _RESUME_CHECK_r22 1205 +#define _RESUME_CHECK_r33 1206 +#define _RETURN_GENERATOR_r01 1207 +#define _RETURN_VALUE_r11 1208 +#define _SAVE_RETURN_OFFSET_r00 1209 +#define _SAVE_RETURN_OFFSET_r11 1210 +#define _SAVE_RETURN_OFFSET_r22 1211 +#define _SAVE_RETURN_OFFSET_r33 1212 +#define _SEND_r22 1213 +#define _SEND_GEN_FRAME_r22 1214 +#define _SETUP_ANNOTATIONS_r00 1215 +#define _SET_ADD_r10 1216 +#define _SET_FUNCTION_ATTRIBUTE_r01 1217 +#define _SET_FUNCTION_ATTRIBUTE_r11 1218 +#define _SET_FUNCTION_ATTRIBUTE_r21 1219 +#define _SET_FUNCTION_ATTRIBUTE_r32 1220 +#define _SET_IP_r00 1221 +#define _SET_IP_r11 1222 +#define _SET_IP_r22 1223 +#define _SET_IP_r33 1224 +#define _SET_UPDATE_r10 1225 +#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r02 1226 +#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r12 1227 +#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r22 1228 +#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r32 1229 +#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03 1230 +#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13 1231 +#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23 1232 +#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33 1233 +#define _SPILL_OR_RELOAD_r01 1234 +#define _SPILL_OR_RELOAD_r02 1235 +#define _SPILL_OR_RELOAD_r03 1236 +#define _SPILL_OR_RELOAD_r10 1237 +#define _SPILL_OR_RELOAD_r12 1238 +#define _SPILL_OR_RELOAD_r13 1239 +#define _SPILL_OR_RELOAD_r20 1240 +#define _SPILL_OR_RELOAD_r21 1241 +#define _SPILL_OR_RELOAD_r23 1242 +#define _SPILL_OR_RELOAD_r30 1243 +#define _SPILL_OR_RELOAD_r31 1244 +#define _SPILL_OR_RELOAD_r32 1245 +#define _START_EXECUTOR_r00 1246 +#define _STORE_ATTR_r20 1247 +#define _STORE_ATTR_INSTANCE_VALUE_r21 1248 +#define _STORE_ATTR_SLOT_r21 1249 +#define _STORE_ATTR_WITH_HINT_r21 1250 +#define _STORE_DEREF_r10 1251 +#define _STORE_FAST_r10 1252 +#define _STORE_FAST_0_r10 1253 +#define _STORE_FAST_1_r10 1254 +#define _STORE_FAST_2_r10 1255 +#define _STORE_FAST_3_r10 1256 +#define _STORE_FAST_4_r10 1257 +#define _STORE_FAST_5_r10 1258 +#define _STORE_FAST_6_r10 1259 +#define _STORE_FAST_7_r10 1260 +#define _STORE_FAST_LOAD_FAST_r11 1261 +#define _STORE_FAST_STORE_FAST_r20 1262 +#define _STORE_GLOBAL_r10 1263 +#define _STORE_NAME_r10 1264 +#define _STORE_SLICE_r30 1265 +#define _STORE_SUBSCR_r30 1266 +#define _STORE_SUBSCR_DICT_r31 1267 +#define _STORE_SUBSCR_LIST_INT_r32 1268 +#define _SWAP_r11 1269 +#define _SWAP_2_r02 1270 +#define _SWAP_2_r12 1271 +#define _SWAP_2_r22 1272 +#define _SWAP_2_r33 1273 +#define _SWAP_3_r03 1274 +#define _SWAP_3_r13 1275 +#define _SWAP_3_r23 1276 +#define _SWAP_3_r33 1277 +#define _TIER2_RESUME_CHECK_r00 1278 +#define _TIER2_RESUME_CHECK_r11 1279 +#define _TIER2_RESUME_CHECK_r22 1280 +#define _TIER2_RESUME_CHECK_r33 1281 +#define _TO_BOOL_r11 1282 +#define _TO_BOOL_BOOL_r01 1283 +#define _TO_BOOL_BOOL_r11 1284 +#define _TO_BOOL_BOOL_r22 1285 +#define _TO_BOOL_BOOL_r33 1286 +#define _TO_BOOL_INT_r11 1287 +#define _TO_BOOL_LIST_r11 1288 +#define _TO_BOOL_NONE_r01 1289 +#define _TO_BOOL_NONE_r11 1290 +#define _TO_BOOL_NONE_r22 1291 +#define _TO_BOOL_NONE_r33 1292 +#define _TO_BOOL_STR_r11 1293 +#define _TRACE_RECORD_r00 1294 +#define _UNARY_INVERT_r11 1295 +#define _UNARY_NEGATIVE_r11 1296 +#define _UNARY_NOT_r01 1297 +#define _UNARY_NOT_r11 1298 +#define _UNARY_NOT_r22 1299 +#define _UNARY_NOT_r33 1300 +#define _UNPACK_EX_r10 1301 +#define _UNPACK_SEQUENCE_r10 1302 +#define _UNPACK_SEQUENCE_LIST_r10 1303 +#define _UNPACK_SEQUENCE_TUPLE_r10 1304 +#define _UNPACK_SEQUENCE_TWO_TUPLE_r12 1305 +#define _WITH_EXCEPT_START_r33 1306 +#define _YIELD_VALUE_r11 1307 +#define MAX_UOP_REGS_ID 1307 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index a61df5642a4d78..6262a14e266c4d 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -204,7 +204,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_COMPARE_OP_FLOAT] = HAS_ARG_FLAG, [_COMPARE_OP_INT] = HAS_ARG_FLAG, [_COMPARE_OP_STR] = HAS_ARG_FLAG, - [_IS_OP] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, + [_IS_OP] = HAS_ARG_FLAG, [_CONTAINS_OP] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_TOS_ANY_SET] = HAS_DEOPT_FLAG, [_CONTAINS_OP_SET] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -1889,11 +1889,11 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { }, }, [_IS_OP] = { - .best = { 2, 2, 2, 2 }, + .best = { 0, 1, 2, 2 }, .entries = { - { -1, -1, -1 }, - { -1, -1, -1 }, - { 1, 2, _IS_OP_r21 }, + { 3, 0, _IS_OP_r03 }, + { 3, 1, _IS_OP_r13 }, + { 3, 2, _IS_OP_r23 }, { -1, -1, -1 }, }, }, @@ -3573,7 +3573,9 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_COMPARE_OP_FLOAT_r23] = _COMPARE_OP_FLOAT, [_COMPARE_OP_INT_r23] = _COMPARE_OP_INT, [_COMPARE_OP_STR_r23] = _COMPARE_OP_STR, - [_IS_OP_r21] = _IS_OP, + [_IS_OP_r03] = _IS_OP, + [_IS_OP_r13] = _IS_OP, + [_IS_OP_r23] = _IS_OP, [_CONTAINS_OP_r21] = _CONTAINS_OP, [_GUARD_TOS_ANY_SET_r01] = _GUARD_TOS_ANY_SET, [_GUARD_TOS_ANY_SET_r11] = _GUARD_TOS_ANY_SET, @@ -4456,7 +4458,9 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_IS_NONE] = "_IS_NONE", [_IS_NONE_r11] = "_IS_NONE_r11", [_IS_OP] = "_IS_OP", - [_IS_OP_r21] = "_IS_OP_r21", + [_IS_OP_r03] = "_IS_OP_r03", + [_IS_OP_r13] = "_IS_OP_r13", + [_IS_OP_r23] = "_IS_OP_r23", [_ITER_CHECK_LIST] = "_ITER_CHECK_LIST", [_ITER_CHECK_LIST_r02] = "_ITER_CHECK_LIST_r02", [_ITER_CHECK_LIST_r12] = "_ITER_CHECK_LIST_r12", diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 62fa02e10d949b..d3c71dd8ae4449 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -3165,6 +3165,71 @@ def testfunc(n): self.assertNotIn("_POP_TOP_INT", uops) self.assertIn("_POP_TOP_NOP", uops) + def test_is_op(self): + def test_is_false(n): + a = object() + b = object() + for _ in range(n): + res = a is b + return res + + res, ex = self._run_with_optimizer(test_is_false, TIER2_THRESHOLD) + self.assertEqual(res, False) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + + self.assertIn("_IS_OP", uops) + self.assertIn("_POP_TOP_NOP", uops) + self.assertNotIn("_POP_TOP", uops) + + + def test_is_true(n): + a = object() + for _ in range(n): + res = a is a + return res + + res, ex = self._run_with_optimizer(test_is_true, TIER2_THRESHOLD) + self.assertEqual(res, True) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + + self.assertIn("_IS_OP", uops) + self.assertIn("_POP_TOP_NOP", uops) + self.assertNotIn("_POP_TOP", uops) + + + def test_is_not(n): + a = object() + b = object() + for _ in range(n): + res = a is not b + return res + + res, ex = self._run_with_optimizer(test_is_not, TIER2_THRESHOLD) + self.assertEqual(res, True) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + + self.assertIn("_IS_OP", uops) + self.assertIn("_POP_TOP_NOP", uops) + self.assertNotIn("_POP_TOP", uops) + + + def test_is_none(n): + a = None + for _ in range(n): + res = a is None + return res + + res, ex = self._run_with_optimizer(test_is_none, TIER2_THRESHOLD) + self.assertEqual(res, True) + self.assertIsNotNone(ex) + + self.assertIn("_IS_OP", uops) + self.assertIn("_POP_TOP_NOP", uops) + self.assertNotIn("_POP_TOP", uops) + def test_143026(self): # https://github.com/python/cpython/issues/143026 diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 43239b129f1b4b..86a4a7f116d93f 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2779,10 +2779,14 @@ dummy_func( // It's always a bool, so we don't care about oparg & 16. } - inst(IS_OP, (left, right -- b)) { + macro(IS_OP) = _IS_OP + POP_TOP + POP_TOP; + + op(_IS_OP, (left, right -- b, l, r)) { int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; - DECREF_INPUTS(); b = res ? PyStackRef_True : PyStackRef_False; + l = left; + r = right; + INPUTS_DEAD(); } family(CONTAINS_OP, INLINE_CACHE_ENTRIES_CONTAINS_OP) = { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 65941f25d897d9..8c8a47d6e134bd 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -9087,34 +9087,78 @@ break; } - case _IS_OP_r21: { + case _IS_OP_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; + _PyStackRef l; + _PyStackRef r; + oparg = CURRENT_OPARG(); + right = stack_pointer[-1]; + left = stack_pointer[-2]; + int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; + b = res ? PyStackRef_True : PyStackRef_False; + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = b; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _IS_OP_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; + _PyStackRef l; + _PyStackRef r; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + right = _stack_item_0; + left = stack_pointer[-1]; + int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; + b = res ? PyStackRef_True : PyStackRef_False; + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; + _tos_cache0 = b; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _IS_OP_r23: { CHECK_CURRENT_CACHED_VALUES(2); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef right; _PyStackRef left; _PyStackRef b; + _PyStackRef l; + _PyStackRef r; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; oparg = CURRENT_OPARG(); right = _stack_item_1; left = _stack_item_0; int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[0] = left; - stack_pointer[1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[0] = left; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); b = res ? PyStackRef_True : PyStackRef_False; + l = left; + r = right; + _tos_cache2 = r; + _tos_cache1 = l; _tos_cache0 = b; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); + SET_CURRENT_CACHED_VALUES(3); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 1a91cdaf6537e7..307ca1fac65e7f 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -7192,25 +7192,36 @@ _PyStackRef left; _PyStackRef right; _PyStackRef b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - b = res ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = b; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyStackRef l; + _PyStackRef r; + _PyStackRef value; + // _IS_OP + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; + b = res ? PyStackRef_True : PyStackRef_False; + l = left; + r = right; + } + // _POP_TOP + { + value = r; + stack_pointer[-2] = b; + stack_pointer[-1] = l; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _POP_TOP + { + value = l; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } DISPATCH(); } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 9f7b2663dacfab..55680c5b824b7b 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -478,8 +478,10 @@ dummy_func(void) { r = right; } - op(_IS_OP, (left, right -- b)) { + op(_IS_OP, (left, right -- b, l, r)) { b = sym_new_type(ctx, &PyBool_Type); + l = left; + r = right; } op(_CONTAINS_OP, (left, right -- b)) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 630dc1703f9ab5..5f4106c33b7f64 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1875,11 +1875,21 @@ } case _IS_OP: { + JitOptRef right; + JitOptRef left; JitOptRef b; + JitOptRef l; + JitOptRef r; + right = stack_pointer[-1]; + left = stack_pointer[-2]; b = sym_new_type(ctx, &PyBool_Type); - CHECK_STACK_BOUNDS(-1); + l = left; + r = right; + CHECK_STACK_BOUNDS(1); stack_pointer[-2] = b; - stack_pointer += -1; + stack_pointer[-1] = l; + stack_pointer[0] = r; + stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } From 9d92ac1225ab93b25acd43b658d214e12c228afe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marta=20G=C3=B3mez=20Mac=C3=ADas?= Date: Sat, 27 Dec 2025 01:36:15 +0100 Subject: [PATCH 582/638] gh-143040: Exit taychon live mode gracefully and display profiled script errors (#143101) --- Lib/profiling/sampling/_sync_coordinator.py | 49 ++++++------- Lib/profiling/sampling/cli.py | 24 ++++--- .../sampling/live_collector/collector.py | 3 + Lib/profiling/sampling/sample.py | 22 +++++- .../test_sampling_profiler/test_cli.py | 1 - .../test_live_collector_ui.py | 72 ++++++++++++++++++- 6 files changed, 133 insertions(+), 38 deletions(-) diff --git a/Lib/profiling/sampling/_sync_coordinator.py b/Lib/profiling/sampling/_sync_coordinator.py index 63d057043f0416..a1cce314b33b19 100644 --- a/Lib/profiling/sampling/_sync_coordinator.py +++ b/Lib/profiling/sampling/_sync_coordinator.py @@ -135,7 +135,7 @@ def _execute_module(module_name: str, module_args: List[str]) -> None: module_args: Arguments to pass to the module Raises: - TargetError: If module execution fails + TargetError: If module cannot be found """ # Replace sys.argv to match how Python normally runs modules # When running 'python -m module args', sys.argv is ["__main__.py", "args"] @@ -145,11 +145,8 @@ def _execute_module(module_name: str, module_args: List[str]) -> None: runpy.run_module(module_name, run_name="__main__", alter_sys=True) except ImportError as e: raise TargetError(f"Module '{module_name}' not found: {e}") from e - except SystemExit: - # SystemExit is normal for modules - pass - except Exception as e: - raise TargetError(f"Error executing module '{module_name}': {e}") from e + # Let other exceptions (including SystemExit) propagate naturally + # so Python prints the full traceback to stderr def _execute_script(script_path: str, script_args: List[str], cwd: str) -> None: @@ -183,22 +180,20 @@ def _execute_script(script_path: str, script_args: List[str], cwd: str) -> None: except PermissionError as e: raise TargetError(f"Permission denied reading script: {script_path}") from e - try: - main_module = types.ModuleType("__main__") - main_module.__file__ = script_path - main_module.__builtins__ = __builtins__ - # gh-140729: Create a __mp_main__ module to allow pickling - sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module + main_module = types.ModuleType("__main__") + main_module.__file__ = script_path + main_module.__builtins__ = __builtins__ + # gh-140729: Create a __mp_main__ module to allow pickling + sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module + try: code = compile(source_code, script_path, 'exec', module='__main__') - exec(code, main_module.__dict__) except SyntaxError as e: raise TargetError(f"Syntax error in script {script_path}: {e}") from e - except SystemExit: - # SystemExit is normal for scripts - pass - except Exception as e: - raise TargetError(f"Error executing script '{script_path}': {e}") from e + + # Execute the script - let exceptions propagate naturally so Python + # prints the full traceback to stderr + exec(code, main_module.__dict__) def main() -> NoReturn: @@ -209,6 +204,8 @@ def main() -> NoReturn: with the sample profiler by signaling when the process is ready to be profiled. """ + # Phase 1: Parse arguments and set up environment + # Errors here are coordinator errors, not script errors try: # Parse and validate arguments sync_port, cwd, target_args = _validate_arguments(sys.argv) @@ -237,21 +234,19 @@ def main() -> NoReturn: # Signal readiness to profiler _signal_readiness(sync_port) - # Execute the target - if is_module: - _execute_module(module_name, module_args) - else: - _execute_script(script_path, script_args, cwd) - except CoordinatorError as e: print(f"Profiler coordinator error: {e}", file=sys.stderr) sys.exit(1) except KeyboardInterrupt: print("Interrupted", file=sys.stderr) sys.exit(1) - except Exception as e: - print(f"Unexpected error in profiler coordinator: {e}", file=sys.stderr) - sys.exit(1) + + # Phase 2: Execute the target script/module + # Let exceptions propagate naturally so Python prints full tracebacks + if is_module: + _execute_module(module_name, module_args) + else: + _execute_script(script_path, script_args, cwd) # Normal exit sys.exit(0) diff --git a/Lib/profiling/sampling/cli.py b/Lib/profiling/sampling/cli.py index dd6431a0322bc7..e43925ea8595f0 100644 --- a/Lib/profiling/sampling/cli.py +++ b/Lib/profiling/sampling/cli.py @@ -272,11 +272,6 @@ def _run_with_sync(original_cmd, suppress_output=False): try: _wait_for_ready_signal(sync_sock, process, _SYNC_TIMEOUT_SEC) - - # Close stderr pipe if we were capturing it - if process.stderr: - process.stderr.close() - except socket.timeout: # If we timeout, kill the process and raise an error if process.poll() is None: @@ -1103,14 +1098,27 @@ def _handle_live_run(args): blocking=args.blocking, ) finally: - # Clean up the subprocess - if process.poll() is None: + # Clean up the subprocess and get any error output + returncode = process.poll() + if returncode is None: + # Process still running - terminate it process.terminate() try: process.wait(timeout=_PROCESS_KILL_TIMEOUT_SEC) except subprocess.TimeoutExpired: process.kill() - process.wait() + # Ensure process is fully terminated + process.wait() + # Read any stderr output (tracebacks, errors, etc.) + if process.stderr: + with process.stderr: + try: + stderr = process.stderr.read() + if stderr: + print(stderr.decode(), file=sys.stderr) + except (OSError, ValueError): + # Ignore errors if pipe is already closed + pass def _handle_replay(args): diff --git a/Lib/profiling/sampling/live_collector/collector.py b/Lib/profiling/sampling/live_collector/collector.py index cdf95a77eeccd8..b31ab060a6b934 100644 --- a/Lib/profiling/sampling/live_collector/collector.py +++ b/Lib/profiling/sampling/live_collector/collector.py @@ -216,6 +216,9 @@ def __init__( def elapsed_time(self): """Get the elapsed time, frozen when finished.""" if self.finished and self.finish_timestamp is not None: + # Handle case where process exited before any samples were collected + if self.start_time is None: + return 0 return self.finish_timestamp - self.start_time return time.perf_counter() - self.start_time if self.start_time else 0 diff --git a/Lib/profiling/sampling/sample.py b/Lib/profiling/sampling/sample.py index 5525bffdf5747d..e73306ebf290e7 100644 --- a/Lib/profiling/sampling/sample.py +++ b/Lib/profiling/sampling/sample.py @@ -42,7 +42,9 @@ def _pause_threads(unwinder, blocking): LiveStatsCollector = None _FREE_THREADED_BUILD = sysconfig.get_config_var("Py_GIL_DISABLED") is not None - +# Minimum number of samples required before showing the TUI +# If fewer samples are collected, we skip the TUI and just print a message +MIN_SAMPLES_FOR_TUI = 200 class SampleProfiler: def __init__(self, pid, sample_interval_usec, all_threads, *, mode=PROFILING_MODE_WALL, native=False, gc=True, opcodes=False, skip_non_matching_threads=True, collect_stats=False, blocking=False): @@ -459,6 +461,11 @@ def sample_live( """ import curses + # Check if process is alive before doing any heavy initialization + if not _is_process_running(pid): + print(f"No samples collected - process {pid} exited before profiling could begin.", file=sys.stderr) + return collector + # Get sample interval from collector sample_interval_usec = collector.sample_interval_usec @@ -486,6 +493,12 @@ def curses_wrapper_func(stdscr): collector.init_curses(stdscr) try: profiler.sample(collector, duration_sec, async_aware=async_aware) + # If too few samples were collected, exit cleanly without showing TUI + if collector.successful_samples < MIN_SAMPLES_FOR_TUI: + # Clear screen before exiting to avoid visual artifacts + stdscr.clear() + stdscr.refresh() + return # Mark as finished and keep the TUI running until user presses 'q' collector.mark_finished() # Keep processing input until user quits @@ -500,4 +513,11 @@ def curses_wrapper_func(stdscr): except KeyboardInterrupt: pass + # If too few samples were collected, print a message + if collector.successful_samples < MIN_SAMPLES_FOR_TUI: + if collector.successful_samples == 0: + print(f"No samples collected - process {pid} exited before profiling could begin.", file=sys.stderr) + else: + print(f"Only {collector.successful_samples} sample(s) collected (minimum {MIN_SAMPLES_FOR_TUI} required for TUI) - process {pid} exited too quickly.", file=sys.stderr) + return collector diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_cli.py b/Lib/test/test_profiling/test_sampling_profiler/test_cli.py index fb4816a0b6085a..f187f6c51d88e2 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_cli.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_cli.py @@ -18,7 +18,6 @@ from profiling.sampling.cli import main from profiling.sampling.errors import SamplingScriptNotFoundError, SamplingModuleNotFoundError, SamplingUnknownProcessError - class TestSampleProfilerCLI(unittest.TestCase): def _setup_sync_mocks(self, mock_socket, mock_popen): """Helper to set up socket and process mocks for coordinator tests.""" diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py b/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py index 2ed9d82a4a4aa2..c0d39f487c8cbd 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py @@ -4,11 +4,14 @@ edge cases, update display, and display helpers. """ +import functools +import io import sys +import tempfile import time import unittest from unittest import mock -from test.support import requires +from test.support import requires, requires_remote_subprocess_debugging from test.support.import_helper import import_module # Only run these tests if curses is available @@ -16,10 +19,12 @@ curses = import_module("curses") from profiling.sampling.live_collector import LiveStatsCollector, MockDisplay +from profiling.sampling.cli import main from ._live_collector_helpers import ( MockThreadInfo, MockInterpreterInfo, ) +from .helpers import close_and_unlink class TestLiveStatsCollectorWithMockDisplay(unittest.TestCase): @@ -816,5 +821,70 @@ def test_get_all_lines_full_display(self): self.assertTrue(any("PID" in line for line in lines)) +@requires_remote_subprocess_debugging() +class TestLiveModeErrors(unittest.TestCase): + """Tests running error commands in the live mode fails gracefully.""" + + def mock_curses_wrapper(self, func): + func(mock.MagicMock()) + + def mock_init_curses_side_effect(self, n_times, mock_self, stdscr): + mock_self.display = MockDisplay() + # Allow the loop to run for a bit (approx 0.5s) before quitting + # This ensures we don't exit too early while the subprocess is + # still failing + for _ in range(n_times): + mock_self.display.simulate_input(-1) + if n_times >= 500: + mock_self.display.simulate_input(ord('q')) + + def test_run_failed_module_live(self): + """Test that running a existing module that fails exits with clean error.""" + + args = [ + "profiling.sampling.cli", "run", "--live", "-m", "test", + "test_asdasd" + ] + + with ( + mock.patch( + 'profiling.sampling.live_collector.collector.LiveStatsCollector.init_curses', + autospec=True, + side_effect=functools.partial(self.mock_init_curses_side_effect, 1000) + ), + mock.patch('curses.wrapper', side_effect=self.mock_curses_wrapper), + mock.patch("sys.argv", args), + mock.patch('sys.stderr', new=io.StringIO()) as fake_stderr + ): + main() + self.assertIn( + 'test test_asdasd crashed -- Traceback (most recent call last):', + fake_stderr.getvalue() + ) + + def test_run_failed_script_live(self): + """Test that running a failing script exits with clean error.""" + script = tempfile.NamedTemporaryFile(suffix=".py") + self.addCleanup(close_and_unlink, script) + script.write(b'1/0\n') + script.seek(0) + + args = ["profiling.sampling.cli", "run", "--live", script.name] + + with ( + mock.patch( + 'profiling.sampling.live_collector.collector.LiveStatsCollector.init_curses', + autospec=True, + side_effect=functools.partial(self.mock_init_curses_side_effect, 200) + ), + mock.patch('curses.wrapper', side_effect=self.mock_curses_wrapper), + mock.patch("sys.argv", args), + mock.patch('sys.stderr', new=io.StringIO()) as fake_stderr + ): + main() + stderr = fake_stderr.getvalue() + self.assertIn('ZeroDivisionError', stderr) + + if __name__ == "__main__": unittest.main() From 54362898f32bf195db898bfead15784d6ab5831b Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sat, 27 Dec 2025 01:39:21 +0000 Subject: [PATCH 583/638] gh-140739: Fix missing exception on allocation failure in BinaryWriter (#143204) --- Modules/_remote_debugging/binary_io_writer.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Modules/_remote_debugging/binary_io_writer.c b/Modules/_remote_debugging/binary_io_writer.c index c8857cec6218be..c129c93efe23c5 100644 --- a/Modules/_remote_debugging/binary_io_writer.c +++ b/Modules/_remote_debugging/binary_io_writer.c @@ -741,6 +741,7 @@ binary_writer_create(const char *filename, uint64_t sample_interval_us, int comp writer->write_buffer = PyMem_Malloc(WRITE_BUFFER_SIZE); if (!writer->write_buffer) { + PyErr_NoMemory(); goto error; } writer->buffer_size = WRITE_BUFFER_SIZE; @@ -753,14 +754,17 @@ binary_writer_create(const char *filename, uint64_t sample_interval_us, int comp NULL /* Use default allocator */ ); if (!writer->string_hash) { + PyErr_NoMemory(); goto error; } writer->strings = PyMem_Malloc(INITIAL_STRING_CAPACITY * sizeof(char *)); if (!writer->strings) { + PyErr_NoMemory(); goto error; } writer->string_lengths = PyMem_Malloc(INITIAL_STRING_CAPACITY * sizeof(size_t)); if (!writer->string_lengths) { + PyErr_NoMemory(); goto error; } writer->string_capacity = INITIAL_STRING_CAPACITY; @@ -773,16 +777,19 @@ binary_writer_create(const char *filename, uint64_t sample_interval_us, int comp NULL /* Use default allocator */ ); if (!writer->frame_hash) { + PyErr_NoMemory(); goto error; } writer->frame_entries = PyMem_Malloc(INITIAL_FRAME_CAPACITY * sizeof(FrameEntry)); if (!writer->frame_entries) { + PyErr_NoMemory(); goto error; } writer->frame_capacity = INITIAL_FRAME_CAPACITY; writer->thread_entries = PyMem_Malloc(INITIAL_THREAD_CAPACITY * sizeof(ThreadEntry)); if (!writer->thread_entries) { + PyErr_NoMemory(); goto error; } writer->thread_capacity = INITIAL_THREAD_CAPACITY; From 5d1e78f7b59ffa3308755b5b2e0f85eb0c6ac890 Mon Sep 17 00:00:00 2001 From: Duane Hilton Date: Sat, 27 Dec 2025 00:23:57 -0700 Subject: [PATCH 584/638] gh-143181: Fix 'overriden' -> 'overridden' in c-api/module.rst (#143182) * Doc: Fix typo 'overriden' -> 'overridden' in c-api/module.rst * Fix 'overriden' -> 'overridden' in tests --- Doc/c-api/module.rst | 2 +- Lib/test/test_build_details.py | 2 +- Lib/test/test_dict.py | 2 +- Lib/test/test_set.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 22f8b1309aa38b..37c92aeb6dcb38 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -571,7 +571,7 @@ A module's token -- and the *your_token* value to use in the above code -- is: of that slot; - For modules created from an ``PyModExport_*`` :ref:`export hook `: the slots array that the export - hook returned (unless overriden with :c:macro:`Py_mod_token`). + hook returned (unless overridden with :c:macro:`Py_mod_token`). .. c:macro:: Py_mod_token diff --git a/Lib/test/test_build_details.py b/Lib/test/test_build_details.py index ba9afe69ba46e8..30d9c213077ab7 100644 --- a/Lib/test/test_build_details.py +++ b/Lib/test/test_build_details.py @@ -33,7 +33,7 @@ class FormatTestsBase: @property def contents(self): - """Install details file contents. Should be overriden by subclasses.""" + """Install details file contents. Should be overridden by subclasses.""" raise NotImplementedError @property diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 665b3e843dd3a5..77a5f2a108d07f 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1581,7 +1581,7 @@ def check_unhashable_key(): with check_unhashable_key(): d.get(key) - # Only TypeError exception is overriden, + # Only TypeError exception is overridden, # other exceptions are left unchanged. class HashError: def __hash__(self): diff --git a/Lib/test/test_set.py b/Lib/test/test_set.py index c0df9507bd7f5e..203a231201c669 100644 --- a/Lib/test/test_set.py +++ b/Lib/test/test_set.py @@ -661,7 +661,7 @@ def check_unhashable_element(): with check_unhashable_element(): myset.discard(elem) - # Only TypeError exception is overriden, + # Only TypeError exception is overridden, # other exceptions are left unchanged. class HashError: def __hash__(self): From 57d569942c6becad85919e3b7fef5f6136c413b0 Mon Sep 17 00:00:00 2001 From: SYan212 Date: Sat, 27 Dec 2025 07:52:28 +0000 Subject: [PATCH 585/638] Fix typos in docs (#143193) --- Doc/library/linecache.rst | 2 +- Doc/tools/extensions/grammar_snippet.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/linecache.rst b/Doc/library/linecache.rst index e766a9280946d3..07305a2a39b252 100644 --- a/Doc/library/linecache.rst +++ b/Doc/library/linecache.rst @@ -31,7 +31,7 @@ The :mod:`linecache` module defines the following functions: .. index:: triple: module; search; path If *filename* indicates a frozen module (starting with ``' Date: Sat, 27 Dec 2025 19:33:56 +0900 Subject: [PATCH 586/638] gh-63016: Add flags parameter on mmap.flush (#139553) Co-authored-by: Victor Stinner --- Doc/library/mmap.rst | 30 ++++++++- Lib/test/test_mmap.py | 9 +++ ...5-10-04-20-48-02.gh-issue-63016.EC9QN_.rst | 1 + Modules/clinic/mmapmodule.c.h | 67 ++++++++++++++++--- Modules/mmapmodule.c | 23 +++++-- 5 files changed, 115 insertions(+), 15 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-04-20-48-02.gh-issue-63016.EC9QN_.rst diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst index 41b90f2c3b3111..28b2d1e244787a 100644 --- a/Doc/library/mmap.rst +++ b/Doc/library/mmap.rst @@ -212,7 +212,7 @@ To map anonymous memory, -1 should be passed as the fileno along with the length Writable :term:`bytes-like object` is now accepted. - .. method:: flush([offset[, size]]) + .. method:: flush([offset[, size]], *, flags=MS_SYNC) Flushes changes made to the in-memory copy of a file back to disk. Without use of this call there is no guarantee that changes are written back before @@ -221,6 +221,12 @@ To map anonymous memory, -1 should be passed as the fileno along with the length whole extent of the mapping is flushed. *offset* must be a multiple of the :const:`PAGESIZE` or :const:`ALLOCATIONGRANULARITY`. + The *flags* parameter specifies the synchronization behavior. + *flags* must be one of the :ref:`MS_* constants ` available + on the system. + + On Windows, the *flags* parameter is ignored. + ``None`` is returned to indicate success. An exception is raised when the call failed. @@ -235,6 +241,9 @@ To map anonymous memory, -1 should be passed as the fileno along with the length specified alone, and the flush operation will extend from *offset* to the end of the mmap. + .. versionchanged:: next + Added *flags* parameter to control synchronization behavior. + .. method:: madvise(option[, start[, length]]) @@ -461,3 +470,22 @@ MAP_* Constants :data:`MAP_TPRO`, :data:`MAP_TRANSLATED_ALLOW_EXECUTE`, and :data:`MAP_UNIX03` constants. +.. _ms-constants: + +MS_* Constants +++++++++++++++ + +.. data:: MS_SYNC + MS_ASYNC + MS_INVALIDATE + + These flags control the synchronization behavior for :meth:`mmap.flush`: + + * :data:`MS_SYNC` - Synchronous flush: writes are scheduled and the call + blocks until they are physically written to storage. + * :data:`MS_ASYNC` - Asynchronous flush: writes are scheduled but the call + returns immediately without waiting for completion. + * :data:`MS_INVALIDATE` - Invalidate cached data: invalidates other mappings + of the same file so they can see the changes. + + .. versionadded:: next diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index aad916ecfe2c27..bc3593ce4ba992 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -1166,6 +1166,15 @@ def test_flush_parameters(self): m.flush(PAGESIZE) m.flush(PAGESIZE, PAGESIZE) + if hasattr(mmap, 'MS_SYNC'): + m.flush(0, PAGESIZE, flags=mmap.MS_SYNC) + if hasattr(mmap, 'MS_ASYNC'): + m.flush(flags=mmap.MS_ASYNC) + if hasattr(mmap, 'MS_INVALIDATE'): + m.flush(PAGESIZE * 2, flags=mmap.MS_INVALIDATE) + if hasattr(mmap, 'MS_ASYNC') and hasattr(mmap, 'MS_INVALIDATE'): + m.flush(0, PAGESIZE, flags=mmap.MS_ASYNC | mmap.MS_INVALIDATE) + @unittest.skipUnless(sys.platform == 'linux', 'Linux only') @support.requires_linux_version(5, 17, 0) def test_set_name(self): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-04-20-48-02.gh-issue-63016.EC9QN_.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-04-20-48-02.gh-issue-63016.EC9QN_.rst new file mode 100644 index 00000000000000..a0aee6ce83a508 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-04-20-48-02.gh-issue-63016.EC9QN_.rst @@ -0,0 +1 @@ +Add a ``flags`` parameter to :meth:`mmap.mmap.flush` to control synchronization behavior. diff --git a/Modules/clinic/mmapmodule.c.h b/Modules/clinic/mmapmodule.c.h index b63f7df2a7e334..db640800ad780f 100644 --- a/Modules/clinic/mmapmodule.c.h +++ b/Modules/clinic/mmapmodule.c.h @@ -2,6 +2,10 @@ preserve [clinic start generated code]*/ +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() @@ -371,29 +375,63 @@ mmap_mmap_tell(PyObject *self, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(mmap_mmap_flush__doc__, -"flush($self, offset=0, size=-1, /)\n" +"flush($self, offset=0, size=-1, /, *, flags=0)\n" "--\n" "\n"); #define MMAP_MMAP_FLUSH_METHODDEF \ - {"flush", _PyCFunction_CAST(mmap_mmap_flush), METH_FASTCALL, mmap_mmap_flush__doc__}, + {"flush", _PyCFunction_CAST(mmap_mmap_flush), METH_FASTCALL|METH_KEYWORDS, mmap_mmap_flush__doc__}, static PyObject * -mmap_mmap_flush_impl(mmap_object *self, Py_ssize_t offset, Py_ssize_t size); +mmap_mmap_flush_impl(mmap_object *self, Py_ssize_t offset, Py_ssize_t size, + int flags); static PyObject * -mmap_mmap_flush(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +mmap_mmap_flush(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(flags), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "", "flags", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "flush", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; Py_ssize_t offset = 0; Py_ssize_t size = -1; + int flags = 0; - if (!_PyArg_CheckPositional("flush", nargs, 0, 2)) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 0, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { goto exit; } if (nargs < 1) { - goto skip_optional; + goto skip_optional_posonly; } + noptargs--; { Py_ssize_t ival = -1; PyObject *iobj = _PyNumber_Index(args[0]); @@ -407,8 +445,9 @@ mmap_mmap_flush(PyObject *self, PyObject *const *args, Py_ssize_t nargs) offset = ival; } if (nargs < 2) { - goto skip_optional; + goto skip_optional_posonly; } + noptargs--; { Py_ssize_t ival = -1; PyObject *iobj = _PyNumber_Index(args[1]); @@ -421,9 +460,17 @@ mmap_mmap_flush(PyObject *self, PyObject *const *args, Py_ssize_t nargs) } size = ival; } -skip_optional: +skip_optional_posonly: + if (!noptargs) { + goto skip_optional_kwonly; + } + flags = PyLong_AsInt(args[2]); + if (flags == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional_kwonly: Py_BEGIN_CRITICAL_SECTION(self); - return_value = mmap_mmap_flush_impl((mmap_object *)self, offset, size); + return_value = mmap_mmap_flush_impl((mmap_object *)self, offset, size, flags); Py_END_CRITICAL_SECTION(); exit: @@ -832,4 +879,4 @@ mmap_mmap_madvise(PyObject *self, PyObject *const *args, Py_ssize_t nargs) #ifndef MMAP_MMAP_MADVISE_METHODDEF #define MMAP_MMAP_MADVISE_METHODDEF #endif /* !defined(MMAP_MMAP_MADVISE_METHODDEF) */ -/*[clinic end generated code: output=fd9ca0ef425af934 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8389e3c8e3db3a78 input=a9049054013a1b77]*/ diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 0928ea6a8b70ba..16e3c0ecefd05d 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -1034,12 +1034,15 @@ mmap.mmap.flush offset: Py_ssize_t = 0 size: Py_ssize_t = -1 / + * + flags: int = 0 [clinic start generated code]*/ static PyObject * -mmap_mmap_flush_impl(mmap_object *self, Py_ssize_t offset, Py_ssize_t size) -/*[clinic end generated code: output=956ced67466149cf input=c50b893bc69520ec]*/ +mmap_mmap_flush_impl(mmap_object *self, Py_ssize_t offset, Py_ssize_t size, + int flags) +/*[clinic end generated code: output=4225f4174dc75a53 input=42ba5fb716b6c294]*/ { CHECK_VALID(NULL); if (size == -1) { @@ -1060,8 +1063,10 @@ mmap_mmap_flush_impl(mmap_object *self, Py_ssize_t offset, Py_ssize_t size) } Py_RETURN_NONE; #elif defined(UNIX) - /* XXX flags for msync? */ - if (-1 == msync(self->data + offset, size, MS_SYNC)) { + if (flags == 0) { + flags = MS_SYNC; + } + if (-1 == msync(self->data + offset, size, flags)) { PyErr_SetFromErrno(PyExc_OSError); return NULL; } @@ -2331,6 +2336,16 @@ mmap_exec(PyObject *module) ADD_INT_MACRO(module, ACCESS_WRITE); ADD_INT_MACRO(module, ACCESS_COPY); +#ifdef MS_INVALIDATE + ADD_INT_MACRO(module, MS_INVALIDATE); +#endif +#ifdef MS_ASYNC + ADD_INT_MACRO(module, MS_ASYNC); +#endif +#ifdef MS_SYNC + ADD_INT_MACRO(module, MS_SYNC); +#endif + #ifdef HAVE_MADVISE // Conventional advice values #ifdef MADV_NORMAL From f5e11facf2d3d89ea8c387376d5889b959c60d82 Mon Sep 17 00:00:00 2001 From: Rafael Fontenelle Date: Sat, 27 Dec 2025 07:48:01 -0300 Subject: [PATCH 587/638] no-issue: Fix override value in os.rst (gh-123522) --- Doc/library/os.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 671270d6112212..f75ad4e67a66d7 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -5993,7 +5993,7 @@ Miscellaneous System Information .. versionchanged:: 3.13 If :option:`-X cpu_count <-X>` is given or :envvar:`PYTHON_CPU_COUNT` is set, - :func:`cpu_count` returns the overridden value *n*. + :func:`cpu_count` returns the override value *n*. .. function:: getloadavg() @@ -6015,7 +6015,7 @@ Miscellaneous System Information in the **system**. If :option:`-X cpu_count <-X>` is given or :envvar:`PYTHON_CPU_COUNT` is set, - :func:`process_cpu_count` returns the overridden value *n*. + :func:`process_cpu_count` returns the override value *n*. See also the :func:`sched_getaffinity` function. From 9976c2b6349a079ae39931d960b8c147e21c6c3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 27 Dec 2025 12:32:52 +0000 Subject: [PATCH 588/638] gh-143195: fix UAF in `{bytearray,memoryview}.hex(sep)` via re-entrant `sep.__len__` (#143209) --- Lib/test/test_bytes.py | 13 +++++++++++++ Lib/test/test_memoryview.py | 14 ++++++++++++++ .../2025-12-27-10-14-26.gh-issue-143195.MNldfr.rst | 3 +++ Objects/bytearrayobject.c | 8 +++++++- Objects/memoryobject.c | 8 +++++++- 5 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-12-27-10-14-26.gh-issue-143195.MNldfr.rst diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 21be61e4fec720..44b16c7d91e996 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -2092,6 +2092,19 @@ def make_case(): with self.assertRaises(BufferError): ba.rsplit(evil) + def test_hex_use_after_free(self): + # Prevent UAF in bytearray.hex(sep) with re-entrant sep.__len__. + # Regression test for https://github.com/python/cpython/issues/143195. + ba = bytearray(b'\xAA') + + class S(bytes): + def __len__(self): + ba.clear() + return 1 + + self.assertRaises(BufferError, ba.hex, S(b':')) + + class AssortedBytesTest(unittest.TestCase): # # Test various combinations of bytes and bytearray diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py index 1bd58eb6408833..51b107103f6836 100644 --- a/Lib/test/test_memoryview.py +++ b/Lib/test/test_memoryview.py @@ -442,6 +442,20 @@ def test_issue22668(self): self.assertEqual(c.format, "H") self.assertEqual(d.format, "H") + def test_hex_use_after_free(self): + # Prevent UAF in memoryview.hex(sep) with re-entrant sep.__len__. + # Regression test for https://github.com/python/cpython/issues/143195. + ba = bytearray(b'A' * 1024) + mv = memoryview(ba) + + class S(bytes): + def __len__(self): + mv.release() + ba.clear() + return 1 + + self.assertRaises(BufferError, mv.hex, S(b':')) + # Variations on source objects for the buffer: bytes-like objects, then arrays # with itemsize > 1. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-12-27-10-14-26.gh-issue-143195.MNldfr.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-27-10-14-26.gh-issue-143195.MNldfr.rst new file mode 100644 index 00000000000000..66dc5e22f0ab23 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-27-10-14-26.gh-issue-143195.MNldfr.rst @@ -0,0 +1,3 @@ +Fix use-after-free crashes in :meth:`bytearray.hex` and :meth:`memoryview.hex` +when the separator's :meth:`~object.__len__` mutates the original object. +Patch by Bénédikt Tran. diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 338c71ad38f7aa..8a454aa48a0930 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -2664,7 +2664,13 @@ bytearray_hex_impl(PyByteArrayObject *self, PyObject *sep, int bytes_per_sep) { char* argbuf = PyByteArray_AS_STRING(self); Py_ssize_t arglen = PyByteArray_GET_SIZE(self); - return _Py_strhex_with_sep(argbuf, arglen, sep, bytes_per_sep); + // Prevent 'self' from being freed if computing len(sep) mutates 'self' + // in _Py_strhex_with_sep(). + // See: https://github.com/python/cpython/issues/143195. + self->ob_exports++; + PyObject *res = _Py_strhex_with_sep(argbuf, arglen, sep, bytes_per_sep); + self->ob_exports--; + return res; } static PyObject * diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index f1232f389210ea..2fd1d784b92ec8 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -2349,7 +2349,13 @@ memoryview_hex_impl(PyMemoryViewObject *self, PyObject *sep, CHECK_RELEASED(self); if (MV_C_CONTIGUOUS(self->flags)) { - return _Py_strhex_with_sep(src->buf, src->len, sep, bytes_per_sep); + // Prevent 'self' from being freed if computing len(sep) mutates 'self' + // in _Py_strhex_with_sep(). + // See: https://github.com/python/cpython/issues/143195. + self->exports++; + PyObject *ret = _Py_strhex_with_sep(src->buf, src->len, sep, bytes_per_sep); + self->exports--; + return ret; } PyBytesWriter *writer = PyBytesWriter_Create(src->len); From 7726119651342bba224e5cc5869859ba7c0416e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 27 Dec 2025 12:57:03 +0000 Subject: [PATCH 589/638] gh-138122: fix AC warnings in `Modules/_remote_debugging/module.c` (#143218) --- Modules/_remote_debugging/module.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Modules/_remote_debugging/module.c b/Modules/_remote_debugging/module.c index 737787a331f948..26ebed13098f0e 100644 --- a/Modules/_remote_debugging/module.c +++ b/Modules/_remote_debugging/module.c @@ -947,6 +947,7 @@ _remote_debugging_RemoteUnwinder_get_stats_impl(RemoteUnwinderObject *self) } /*[clinic input] +@permit_long_docstring_body @critical_section _remote_debugging.RemoteUnwinder.pause_threads @@ -963,7 +964,7 @@ Returns True if threads were successfully paused, False if they were already pau static PyObject * _remote_debugging_RemoteUnwinder_pause_threads_impl(RemoteUnwinderObject *self) -/*[clinic end generated code: output=aaf2bdc0a725750c input=78601c60dbc245fe]*/ +/*[clinic end generated code: output=aaf2bdc0a725750c input=d8a266f19a81c67e]*/ { #ifdef Py_REMOTE_DEBUG_SUPPORTS_BLOCKING if (self->threads_stopped) { @@ -985,6 +986,7 @@ _remote_debugging_RemoteUnwinder_pause_threads_impl(RemoteUnwinderObject *self) } /*[clinic input] +@permit_long_docstring_body @critical_section _remote_debugging.RemoteUnwinder.resume_threads @@ -997,7 +999,7 @@ Returns True if threads were successfully resumed, False if they were not paused static PyObject * _remote_debugging_RemoteUnwinder_resume_threads_impl(RemoteUnwinderObject *self) -/*[clinic end generated code: output=8d6781ea37095536 input=67ca813bd804289e]*/ +/*[clinic end generated code: output=8d6781ea37095536 input=16baaaab007f4259]*/ { #ifdef Py_REMOTE_DEBUG_SUPPORTS_BLOCKING if (!self->threads_stopped) { @@ -1261,6 +1263,7 @@ class _remote_debugging.BinaryWriter "BinaryWriterObject *" "&PyBinaryWriter_Typ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=e948838b90a2003c]*/ /*[clinic input] +@permit_long_docstring_body _remote_debugging.BinaryWriter.__init__ filename: str sample_interval_us: unsigned_long_long @@ -1285,7 +1288,7 @@ _remote_debugging_BinaryWriter___init___impl(BinaryWriterObject *self, unsigned long long sample_interval_us, unsigned long long start_time_us, int compression) -/*[clinic end generated code: output=014c0306f1bacf4b input=57497fe3cb9214a6]*/ +/*[clinic end generated code: output=014c0306f1bacf4b input=3bdf01c1cc2f5a1d]*/ { if (self->writer) { binary_writer_destroy(self->writer); @@ -1300,6 +1303,7 @@ _remote_debugging_BinaryWriter___init___impl(BinaryWriterObject *self, } /*[clinic input] +@permit_long_docstring_body _remote_debugging.BinaryWriter.write_sample stack_frames: object timestamp_us: unsigned_long_long @@ -1315,7 +1319,7 @@ static PyObject * _remote_debugging_BinaryWriter_write_sample_impl(BinaryWriterObject *self, PyObject *stack_frames, unsigned long long timestamp_us) -/*[clinic end generated code: output=24d5b86679b4128f input=dce3148417482624]*/ +/*[clinic end generated code: output=24d5b86679b4128f input=4e6d832d360bea46]*/ { if (!self->writer) { PyErr_SetString(PyExc_ValueError, "Writer is closed"); @@ -1422,6 +1426,7 @@ _remote_debugging_BinaryWriter___exit___impl(BinaryWriterObject *self, } /*[clinic input] +@permit_long_docstring_body _remote_debugging.BinaryWriter.get_stats Get encoding statistics for the writer. @@ -1432,7 +1437,7 @@ record counts, frames written/saved, and compression ratio. static PyObject * _remote_debugging_BinaryWriter_get_stats_impl(BinaryWriterObject *self) -/*[clinic end generated code: output=06522cd52544df89 input=82968491b53ad277]*/ +/*[clinic end generated code: output=06522cd52544df89 input=837c874ffdebd24c]*/ { if (!self->writer) { PyErr_SetString(PyExc_ValueError, "Writer is closed"); @@ -1754,6 +1759,7 @@ _remote_debugging_zstd_available_impl(PyObject *module) * ============================================================================ */ /*[clinic input] +@permit_long_docstring_body _remote_debugging.get_child_pids pid: int @@ -1779,7 +1785,7 @@ Child processes may exit or new ones may be created after the list is returned. static PyObject * _remote_debugging_get_child_pids_impl(PyObject *module, int pid, int recursive) -/*[clinic end generated code: output=1ae2289c6b953e4b input=3395cbe7f17066c9]*/ +/*[clinic end generated code: output=1ae2289c6b953e4b input=19d8d5d6e2b59e6e]*/ { return enumerate_child_pids((pid_t)pid, recursive); } From 00e24b80e092e7d36dc189fd260b2a4e730a6e7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 27 Dec 2025 13:12:03 +0000 Subject: [PATCH 590/638] gh-142664: fix UAF in `memoryview.__hash__` via re-entrant data's `__hash__` (#143217) --- Lib/test/test_memoryview.py | 14 ++++++++++++++ .../2025-12-27-13-18-12.gh-issue-142664.peeEDV.rst | 3 +++ Objects/memoryobject.c | 13 ++++++++++--- 3 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-12-27-13-18-12.gh-issue-142664.peeEDV.rst diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py index 51b107103f6836..656318668e6d6e 100644 --- a/Lib/test/test_memoryview.py +++ b/Lib/test/test_memoryview.py @@ -387,6 +387,20 @@ def test_hash_writable(self): m = self._view(b) self.assertRaises(ValueError, hash, m) + def test_hash_use_after_free(self): + # Prevent crash in memoryview(v).__hash__ with re-entrant v.__hash__. + # Regression test for https://github.com/python/cpython/issues/142664. + class E(array.array): + def __hash__(self): + mv.release() + self.clear() + return 123 + + v = E('B', b'A' * 4096) + mv = memoryview(v).toreadonly() # must be read-only for hash() + self.assertRaises(BufferError, hash, mv) + self.assertRaises(BufferError, mv.__hash__) + def test_weakref(self): # Check memoryviews are weakrefable for tp in self._types: diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-12-27-13-18-12.gh-issue-142664.peeEDV.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-27-13-18-12.gh-issue-142664.peeEDV.rst new file mode 100644 index 00000000000000..39c218395cc4d3 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-27-13-18-12.gh-issue-142664.peeEDV.rst @@ -0,0 +1,3 @@ +Fix a use-after-free crash in :meth:`memoryview.__hash__ ` +when the ``__hash__`` method of the referenced object mutates that object or +the view. Patch by Bénédikt Tran. diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index 2fd1d784b92ec8..f50de3e4c64071 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -3231,9 +3231,16 @@ memory_hash(PyObject *_self) "memoryview: hashing is restricted to formats 'B', 'b' or 'c'"); return -1; } - if (view->obj != NULL && PyObject_Hash(view->obj) == -1) { - /* Keep the original error message */ - return -1; + if (view->obj != NULL) { + // Prevent 'self' from being freed when computing the item's hash. + // See https://github.com/python/cpython/issues/142664. + self->exports++; + int rc = PyObject_Hash(view->obj); + self->exports--; + if (rc == -1) { + /* Keep the original error message */ + return -1; + } } if (!MV_C_CONTIGUOUS(self->flags)) { From 3a728e5f93c1e4c125406eeeb76d5df1c1726409 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sat, 27 Dec 2025 13:38:11 +0000 Subject: [PATCH 591/638] gh-131591: Do not free page caches that weren't allocated (#143205) --- Python/remote_debug.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Python/remote_debug.h b/Python/remote_debug.h index d9c5c480fe9a86..dba6da3bad4197 100644 --- a/Python/remote_debug.h +++ b/Python/remote_debug.h @@ -154,7 +154,9 @@ static void _Py_RemoteDebug_FreePageCache(proc_handle_t *handle) { for (int i = 0; i < MAX_PAGES; i++) { - PyMem_RawFree(handle->pages[i].data); + if (handle->pages[i].data) { + PyMem_RawFree(handle->pages[i].data); + } handle->pages[i].data = NULL; handle->pages[i].valid = 0; } From 84fcdbd86ecd81f7cc793e22268a029ac6cf29c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 27 Dec 2025 14:30:09 +0000 Subject: [PATCH 592/638] gh-142664: fix `PyObject_Hash` invokation post GH-143217 (#143223) --- Objects/memoryobject.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index f50de3e4c64071..f3b7e4a396b4a1 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -3235,9 +3235,9 @@ memory_hash(PyObject *_self) // Prevent 'self' from being freed when computing the item's hash. // See https://github.com/python/cpython/issues/142664. self->exports++; - int rc = PyObject_Hash(view->obj); + Py_hash_t h = PyObject_Hash(view->obj); self->exports--; - if (rc == -1) { + if (h == -1) { /* Keep the original error message */ return -1; } From 61ee04834b096be00678c6819b4957f3f4413a5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 27 Dec 2025 14:57:13 +0000 Subject: [PATCH 593/638] gh-142557: fix UAF in `bytearray.__mod__` when object is mutated while formatting `%`-style arguments (#143213) --- Lib/test/test_bytes.py | 12 ++++++++++++ .../2025-12-27-12-25-06.gh-issue-142557.KWOc8b.rst | 3 +++ Objects/bytearrayobject.c | 10 +++++++++- 3 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-12-27-12-25-06.gh-issue-142557.KWOc8b.rst diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 44b16c7d91e996..e0baeece34c7b3 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1382,6 +1382,18 @@ def test_bytearray_api(self): except OSError: pass + def test_mod_concurrent_mutation(self): + # Prevent crash in __mod__ when formatting mutates the bytearray. + # Regression test for https://github.com/python/cpython/issues/142557. + fmt = bytearray(b"%a end") + + class S: + def __repr__(self): + fmt.clear() + return "E" + + self.assertRaises(BufferError, fmt.__mod__, S()) + def test_reverse(self): b = bytearray(b'hello') self.assertEqual(b.reverse(), None) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-12-27-12-25-06.gh-issue-142557.KWOc8b.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-27-12-25-06.gh-issue-142557.KWOc8b.rst new file mode 100644 index 00000000000000..b7f7a585906c34 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-27-12-25-06.gh-issue-142557.KWOc8b.rst @@ -0,0 +1,3 @@ +Fix a use-after-free crash in :ref:`bytearray.__mod__ ` when +the :class:`!bytearray` is mutated while formatting the ``%``-style arguments. +Patch by Bénédikt Tran. diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 8a454aa48a0930..5262ac20c07300 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -2843,7 +2843,15 @@ bytearray_mod_lock_held(PyObject *v, PyObject *w) _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(v); if (!PyByteArray_Check(v)) Py_RETURN_NOTIMPLEMENTED; - return _PyBytes_FormatEx(PyByteArray_AS_STRING(v), PyByteArray_GET_SIZE(v), w, 1); + + PyByteArrayObject *self = _PyByteArray_CAST(v); + /* Increase exports to prevent bytearray storage from changing during op. */ + self->ob_exports++; + PyObject *res = _PyBytes_FormatEx( + PyByteArray_AS_STRING(v), PyByteArray_GET_SIZE(v), w, 1 + ); + self->ob_exports--; + return res; } static PyObject * From 23abbf1f2b9123c9c486485ea37da6d36b464f88 Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Sun, 28 Dec 2025 20:15:24 +0800 Subject: [PATCH 594/638] gh-139922: Link to results in MSVC tail calling in What's New 3.15 (GH-143242) Link to results in MSVC tail calling for whats new in 3.15 --- Doc/whatsnew/3.15.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 0d35eed38f303d..11f08031ec54f2 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -854,11 +854,13 @@ Optimizations * Builds using Visual Studio 2026 (MSVC 18) may now use the new :ref:`tail-calling interpreter `. - Results on an early experimental MSVC compiler reported roughly 15% speedup - on the geometric mean of pyperformance on Windows x86-64 over - the switch-case interpreter. We have - observed speedups ranging from 15% for large pure-Python libraries + Results on Visual Studio 18.1.1 report between + `15-20% `__ + speedup on the geometric mean of pyperformance on Windows x86-64 over + the switch-case interpreter on an AMD Ryzen 7 5800X. We have + observed speedups ranging from 14% for large pure-Python libraries to 40% for long-running small pure-Python scripts on Windows. + This was made possible by a new feature introduced in MSVC 18. (Contributed by Chris Eibl, Ken Jin, and Brandt Bucher in :gh:`143068`. Special thanks to the MSVC team including Hulon Jenkins.) From 522563549a49d28e763635c58274a23a6055f041 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 28 Dec 2025 14:30:36 +0200 Subject: [PATCH 595/638] gh-143003: Fix possible shared buffer overflow in bytearray.extend() (GH-143086) When __length_hint__() returns 0 for non-empty iterator, the data can be written past the shared 0-terminated buffer, corrupting it. --- Lib/test/test_bytes.py | 17 +++++++++++++++++ ...25-12-23-00-13-02.gh-issue-143003.92g5qW.rst | 2 ++ Objects/bytearrayobject.c | 4 ++-- 3 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-12-23-00-13-02.gh-issue-143003.92g5qW.rst diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index e0baeece34c7b3..c42c0d4f5e9bc2 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -2104,6 +2104,23 @@ def make_case(): with self.assertRaises(BufferError): ba.rsplit(evil) + def test_extend_empty_buffer_overflow(self): + # gh-143003 + class EvilIter: + def __iter__(self): + return self + def __next__(self): + return next(source) + def __length_hint__(self): + return 0 + + # Use ASCII digits so float() takes the fast path that expects a NUL terminator. + source = iter(b'42') + ba = bytearray() + ba.extend(EvilIter()) + + self.assertRaises(ValueError, float, bytearray()) + def test_hex_use_after_free(self): # Prevent UAF in bytearray.hex(sep) with re-entrant sep.__len__. # Regression test for https://github.com/python/cpython/issues/143195. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-12-23-00-13-02.gh-issue-143003.92g5qW.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-23-00-13-02.gh-issue-143003.92g5qW.rst new file mode 100644 index 00000000000000..30df3c53abd29f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-23-00-13-02.gh-issue-143003.92g5qW.rst @@ -0,0 +1,2 @@ +Fix an overflow of the shared empty buffer in :meth:`bytearray.extend` when +``__length_hint__()`` returns 0 for non-empty iterator. diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 5262ac20c07300..7f09769e12f05f 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -2223,7 +2223,6 @@ bytearray_extend_impl(PyByteArrayObject *self, PyObject *iterable_of_ints) Py_DECREF(bytearray_obj); return NULL; } - buf[len++] = value; Py_DECREF(item); if (len >= buf_size) { @@ -2233,7 +2232,7 @@ bytearray_extend_impl(PyByteArrayObject *self, PyObject *iterable_of_ints) Py_DECREF(bytearray_obj); return PyErr_NoMemory(); } - addition = len >> 1; + addition = len ? len >> 1 : 1; if (addition > PyByteArray_SIZE_MAX - len) buf_size = PyByteArray_SIZE_MAX; else @@ -2247,6 +2246,7 @@ bytearray_extend_impl(PyByteArrayObject *self, PyObject *iterable_of_ints) have invalidated it. */ buf = PyByteArray_AS_STRING(bytearray_obj); } + buf[len++] = value; } Py_DECREF(it); From 836b2810d501fafdefb619e282c745e7d1dfa90f Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 28 Dec 2025 12:52:32 +0000 Subject: [PATCH 596/638] gh-136186: Fix more flaky tests in test_external_inspection (#143235) --- Lib/test/test_external_inspection.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index b1a3a8e65a9802..fe1b5fbe00bbc4 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -1752,7 +1752,12 @@ def main_work(): unwinder_gil = RemoteUnwinder( p.pid, only_active_thread=True ) - gil_traces = _get_stack_trace_with_retry(unwinder_gil) + # Use condition to retry until we capture a thread holding the GIL + # (sampling may catch moments with no GIL holder on slow CI) + gil_traces = _get_stack_trace_with_retry( + unwinder_gil, + condition=lambda t: sum(len(i.threads) for i in t) >= 1, + ) # Count threads total_threads = sum( From 3ccc76f036bfaabb5a4631783b966501fe64859a Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 28 Dec 2025 13:50:23 +0000 Subject: [PATCH 597/638] gh-143228: Fix UAF in perf trampoline during finalization (#143233) --- Include/internal/pycore_ceval.h | 1 - Include/internal/pycore_interp_structs.h | 4 +- ...-12-27-23-57-43.gh-issue-143228.m3EF9E.rst | 4 ++ Python/perf_trampoline.c | 67 ++++++++++++++++--- Python/pylifecycle.c | 1 - 5 files changed, 66 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-12-27-23-57-43.gh-issue-143228.m3EF9E.rst diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index c6c82038d7c85f..f6bdba3e9916c0 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -103,7 +103,6 @@ extern int _PyPerfTrampoline_SetCallbacks(_PyPerf_Callbacks *); extern void _PyPerfTrampoline_GetCallbacks(_PyPerf_Callbacks *); extern int _PyPerfTrampoline_Init(int activate); extern int _PyPerfTrampoline_Fini(void); -extern void _PyPerfTrampoline_FreeArenas(void); extern int _PyIsPerfTrampolineActive(void); extern PyStatus _PyPerfTrampoline_AfterFork_Child(void); #ifdef PY_HAVE_PERF_TRAMPOLINE diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 818c4f159591fe..3fe1fdaa1589b6 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -87,7 +87,9 @@ struct _ceval_runtime_state { struct trampoline_api_st trampoline_api; FILE *map_file; Py_ssize_t persist_after_fork; - _PyFrameEvalFunction prev_eval_frame; + _PyFrameEvalFunction prev_eval_frame; + Py_ssize_t trampoline_refcount; + int code_watcher_id; #else int _not_used; #endif diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-12-27-23-57-43.gh-issue-143228.m3EF9E.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-27-23-57-43.gh-issue-143228.m3EF9E.rst new file mode 100644 index 00000000000000..893bc29543d91d --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-27-23-57-43.gh-issue-143228.m3EF9E.rst @@ -0,0 +1,4 @@ +Fix use-after-free in perf trampoline when toggling profiling while +threads are running or during interpreter finalization with daemon threads +active. The fix uses reference counting to ensure trampolines are not freed +while any code object could still reference them. Pach by Pablo Galindo diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index 335d8ac7dadd10..c0dc1f7a49bdca 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -204,6 +204,42 @@ enum perf_trampoline_type { #define persist_after_fork _PyRuntime.ceval.perf.persist_after_fork #define perf_trampoline_type _PyRuntime.ceval.perf.perf_trampoline_type #define prev_eval_frame _PyRuntime.ceval.perf.prev_eval_frame +#define trampoline_refcount _PyRuntime.ceval.perf.trampoline_refcount +#define code_watcher_id _PyRuntime.ceval.perf.code_watcher_id + +static void free_code_arenas(void); + +static void +perf_trampoline_reset_state(void) +{ + free_code_arenas(); + if (code_watcher_id >= 0) { + PyCode_ClearWatcher(code_watcher_id); + code_watcher_id = -1; + } + extra_code_index = -1; +} + +static int +perf_trampoline_code_watcher(PyCodeEvent event, PyCodeObject *co) +{ + if (event != PY_CODE_EVENT_DESTROY) { + return 0; + } + if (extra_code_index == -1) { + return 0; + } + py_trampoline f = NULL; + int ret = _PyCode_GetExtra((PyObject *)co, extra_code_index, (void **)&f); + if (ret != 0 || f == NULL) { + return 0; + } + trampoline_refcount--; + if (trampoline_refcount == 0) { + perf_trampoline_reset_state(); + } + return 0; +} static void perf_map_write_entry(void *state, const void *code_addr, @@ -407,6 +443,7 @@ py_trampoline_evaluator(PyThreadState *ts, _PyInterpreterFrame *frame, perf_code_arena->code_size, co); _PyCode_SetExtra((PyObject *)co, extra_code_index, (void *)new_trampoline); + trampoline_refcount++; f = new_trampoline; } assert(f != NULL); @@ -433,6 +470,7 @@ int PyUnstable_PerfTrampoline_CompileCode(PyCodeObject *co) } trampoline_api.write_state(trampoline_api.state, new_trampoline, perf_code_arena->code_size, co); + trampoline_refcount++; return _PyCode_SetExtra((PyObject *)co, extra_code_index, (void *)new_trampoline); } @@ -487,6 +525,10 @@ _PyPerfTrampoline_Init(int activate) { #ifdef PY_HAVE_PERF_TRAMPOLINE PyThreadState *tstate = _PyThreadState_GET(); + if (code_watcher_id == 0) { + // Initialize to -1 since 0 is a valid watcher ID + code_watcher_id = -1; + } if (!activate) { _PyInterpreterState_SetEvalFrameFunc(tstate->interp, prev_eval_frame); perf_status = PERF_STATUS_NO_INIT; @@ -504,6 +546,13 @@ _PyPerfTrampoline_Init(int activate) if (new_code_arena() < 0) { return -1; } + code_watcher_id = PyCode_AddWatcher(perf_trampoline_code_watcher); + if (code_watcher_id < 0) { + PyErr_FormatUnraisable("Failed to register code watcher for perf trampoline"); + free_code_arenas(); + return -1; + } + trampoline_refcount = 1; // Base refcount held by the system perf_status = PERF_STATUS_OK; } #endif @@ -525,17 +574,19 @@ _PyPerfTrampoline_Fini(void) trampoline_api.free_state(trampoline_api.state); perf_trampoline_type = PERF_TRAMPOLINE_UNSET; } - extra_code_index = -1; + + // Prevent new trampolines from being created perf_status = PERF_STATUS_NO_INIT; -#endif - return 0; -} -void _PyPerfTrampoline_FreeArenas(void) { -#ifdef PY_HAVE_PERF_TRAMPOLINE - free_code_arenas(); + // Decrement base refcount. If refcount reaches 0, all code objects are already + // dead so clean up now. Otherwise, watcher remains active to clean up when last + // code object dies; extra_code_index stays valid so watcher can identify them. + trampoline_refcount--; + if (trampoline_refcount == 0) { + perf_trampoline_reset_state(); + } #endif - return; + return 0; } int diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 45b585faf9c980..bb663db195c089 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1944,7 +1944,6 @@ finalize_interp_clear(PyThreadState *tstate) _PyArg_Fini(); _Py_ClearFileSystemEncoding(); _PyPerfTrampoline_Fini(); - _PyPerfTrampoline_FreeArenas(); } finalize_interp_types(tstate->interp); From 3ca1f2a370e44874d0dc8c82a01465e0171bec5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fatih=20=C3=87elik?= Date: Sun, 28 Dec 2025 17:48:43 +0300 Subject: [PATCH 598/638] gh-143241: Fix infinite loop in `zoneinfo._common.load_data` (#143243) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Correctly reject truncated TZif files in `ZoneInfo.from_file`. --------- Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Doc/library/zoneinfo.rst | 3 +++ Lib/test/test_zoneinfo/test_zoneinfo.py | 2 ++ Lib/zoneinfo/_common.py | 9 ++++----- .../2025-12-28-13-49-06.gh-issue-143241.5H4b8d.rst | 2 ++ 4 files changed, 11 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-12-28-13-49-06.gh-issue-143241.5H4b8d.rst diff --git a/Doc/library/zoneinfo.rst b/Doc/library/zoneinfo.rst index 8147e58d322667..759ec4277b8b7d 100644 --- a/Doc/library/zoneinfo.rst +++ b/Doc/library/zoneinfo.rst @@ -206,6 +206,9 @@ The ``ZoneInfo`` class has two alternate constructors: Objects created via this constructor cannot be pickled (see `pickling`_). + :exc:`ValueError` is raised if the data read from *file_obj* is not a valid + TZif file. + .. classmethod:: ZoneInfo.no_cache(key) An alternate constructor that bypasses the constructor's cache. It is diff --git a/Lib/test/test_zoneinfo/test_zoneinfo.py b/Lib/test/test_zoneinfo/test_zoneinfo.py index 8f3ca59c9ef5ed..581072d0701d65 100644 --- a/Lib/test/test_zoneinfo/test_zoneinfo.py +++ b/Lib/test/test_zoneinfo/test_zoneinfo.py @@ -252,6 +252,8 @@ def test_bad_zones(self): bad_zones = [ b"", # Empty file b"AAAA3" + b" " * 15, # Bad magic + # Truncated V2 file (should not loop indefinitely) + b"TZif2" + (b"\x00" * 39) + b"TZif2" + (b"\x00" * 39) + b"\n" + b"Part", ] for bad_zone in bad_zones: diff --git a/Lib/zoneinfo/_common.py b/Lib/zoneinfo/_common.py index 03cc42149f9b74..59f3f0ce853f74 100644 --- a/Lib/zoneinfo/_common.py +++ b/Lib/zoneinfo/_common.py @@ -118,11 +118,10 @@ def get_abbr(idx): c = fobj.read(1) # Should be \n assert c == b"\n", c - tz_bytes = b"" - while (c := fobj.read(1)) != b"\n": - tz_bytes += c - - tz_str = tz_bytes + line = fobj.readline() + if not line.endswith(b"\n"): + raise ValueError("Invalid TZif file: unexpected end of file") + tz_str = line.rstrip(b"\n") else: tz_str = None diff --git a/Misc/NEWS.d/next/Library/2025-12-28-13-49-06.gh-issue-143241.5H4b8d.rst b/Misc/NEWS.d/next/Library/2025-12-28-13-49-06.gh-issue-143241.5H4b8d.rst new file mode 100644 index 00000000000000..7170a06015ee7c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-28-13-49-06.gh-issue-143241.5H4b8d.rst @@ -0,0 +1,2 @@ +:mod:`zoneinfo`: fix infinite loop in :meth:`ZoneInfo.from_file +` when parsing a malformed TZif file. Patch by Fatih Celik. From c3bfe5d5aa557e98b9ab53b8dbe9887c8c80be35 Mon Sep 17 00:00:00 2001 From: AN Long Date: Mon, 29 Dec 2025 00:36:52 +0900 Subject: [PATCH 599/638] gh-63016: fix failing `mmap.flush` tests on FreeBSD (#143230) Fix `mmap.flush` tests introduced in 1af21ea32043ad5bd4eaacd48a1718d4e0bef945 where some flag combinations are not supported on FreeBSD. --- Lib/test/test_mmap.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index bc3593ce4ba992..48bf246cadd2f8 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -1173,7 +1173,13 @@ def test_flush_parameters(self): if hasattr(mmap, 'MS_INVALIDATE'): m.flush(PAGESIZE * 2, flags=mmap.MS_INVALIDATE) if hasattr(mmap, 'MS_ASYNC') and hasattr(mmap, 'MS_INVALIDATE'): - m.flush(0, PAGESIZE, flags=mmap.MS_ASYNC | mmap.MS_INVALIDATE) + if sys.platform == 'freebsd': + # FreeBSD doesn't support this combination + with self.assertRaises(OSError) as cm: + m.flush(0, PAGESIZE, flags=mmap.MS_ASYNC | mmap.MS_INVALIDATE) + self.assertEqual(cm.exception.errno, errno.EINVAL) + else: + m.flush(0, PAGESIZE, flags=mmap.MS_ASYNC | mmap.MS_INVALIDATE) @unittest.skipUnless(sys.platform == 'linux', 'Linux only') @support.requires_linux_version(5, 17, 0) From fa9a4254e81c0abcc3345021c45aaf5f788f9ea9 Mon Sep 17 00:00:00 2001 From: Prithviraj Chaudhuri Date: Sun, 28 Dec 2025 11:57:44 -0500 Subject: [PATCH 600/638] gh-142195: Fixed Popen.communicate indefinite loops (GH-143203) Changed condition to evaluate if timeout is less than or equals to 0. This is needed for simulated time environments such as Shadow where the time will match exactly on the boundary. --------- Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> --- Lib/subprocess.py | 2 +- .../next/Library/2025-12-27-00-14-56.gh-issue-142195.UgBEo5.rst | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-12-27-00-14-56.gh-issue-142195.UgBEo5.rst diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 17333d8c02255d..3cebd7883fcf29 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -2140,7 +2140,7 @@ def _communicate(self, input, endtime, orig_timeout): while selector.get_map(): timeout = self._remaining_time(endtime) - if timeout is not None and timeout < 0: + if timeout is not None and timeout <= 0: self._check_timeout(endtime, orig_timeout, stdout, stderr, skip_check_and_raise=True) diff --git a/Misc/NEWS.d/next/Library/2025-12-27-00-14-56.gh-issue-142195.UgBEo5.rst b/Misc/NEWS.d/next/Library/2025-12-27-00-14-56.gh-issue-142195.UgBEo5.rst new file mode 100644 index 00000000000000..b2b1ffe7225bd7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-27-00-14-56.gh-issue-142195.UgBEo5.rst @@ -0,0 +1 @@ +Updated timeout evaluation logic in :mod:`subprocess` to be compatible with deterministic environments like Shadow where time moves exactly as requested. From 0efbad60e13cbc8b27a5ca3a5d9afcdcc957b19e Mon Sep 17 00:00:00 2001 From: wangxiaolei Date: Mon, 29 Dec 2025 02:03:30 +0800 Subject: [PATCH 601/638] gh-142994, gh-142996: document missing async generator and coroutine field entries in `inspect` (#142997) --- Doc/library/inspect.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index e5abd68f03b9c3..9e53dd70ab564e 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -273,6 +273,9 @@ attributes (see :ref:`import-mod-attrs` for module attributes): +-----------------+-------------------+---------------------------+ | | ag_running | is the generator running? | +-----------------+-------------------+---------------------------+ +| | ag_suspended | is the generator | +| | | suspended? | ++-----------------+-------------------+---------------------------+ | | ag_code | code | +-----------------+-------------------+---------------------------+ | coroutine | __name__ | name | @@ -286,6 +289,9 @@ attributes (see :ref:`import-mod-attrs` for module attributes): +-----------------+-------------------+---------------------------+ | | cr_running | is the coroutine running? | +-----------------+-------------------+---------------------------+ +| | cr_suspended | is the coroutine | +| | | suspended? | ++-----------------+-------------------+---------------------------+ | | cr_code | code | +-----------------+-------------------+---------------------------+ | | cr_origin | where coroutine was | @@ -319,6 +325,18 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Add ``__builtins__`` attribute to functions. +.. versionchanged:: 3.11 + + Add ``gi_suspended`` attribute to generators. + +.. versionchanged:: 3.11 + + Add ``cr_suspended`` attribute to coroutines. + +.. versionchanged:: 3.12 + + Add ``ag_suspended`` attribute to async generators. + .. versionchanged:: 3.14 Add ``f_generator`` attribute to frames. From c3febba73b05bb15b15930d545b479a3245cfe11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Simon?= Date: Sun, 28 Dec 2025 20:06:06 +0100 Subject: [PATCH 602/638] gh-140870: Full coverage for _pyrepl._module_completer (#143244) Full coverage for _pyrepl._module_completer Co-authored-by: Tomas R. --- Lib/test/test_pyrepl/test_pyrepl.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index ddcaafc9b7dbe8..65a252c95e5842 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -1037,6 +1037,8 @@ def test_relative_import_completions(self): (None, "from . import readl\t\n", "from . import readl"), ("_pyrepl", "from .readl\t\n", "from .readline"), ("_pyrepl", "from . import readl\t\n", "from . import readline"), + ("_pyrepl", "from .. import toodeep\t\n", "from .. import toodeep"), + ("concurrent", "from .futures.i\t\n", "from .futures.interpreter"), ) for package, code, expected in cases: with self.subTest(code=code): @@ -1075,6 +1077,18 @@ def test_no_fallback_on_regular_completion(self): output = reader.readline() self.assertEqual(output, expected) + def test_global_cache(self): + with (tempfile.TemporaryDirectory() as _dir1, + patch.object(sys, "path", [_dir1, *sys.path])): + dir1 = pathlib.Path(_dir1) + (dir1 / "mod_aa.py").mkdir() + (dir1 / "mod_bb.py").mkdir() + events = code_to_events("import mod_a\t\nimport mod_b\t\n") + reader = self.prepare_reader(events, namespace={}) + output_1, output_2 = reader.readline(), reader.readline() + self.assertEqual(output_1, "import mod_aa") + self.assertEqual(output_2, "import mod_bb") + def test_hardcoded_stdlib_submodules(self): cases = ( ("import collections.\t\n", "import collections.abc"), @@ -1203,6 +1217,7 @@ def test_parse_error(self): 'import ..foo', 'import .foo.bar', 'import foo; x = 1', + 'import foo; 1,', 'import a.; x = 1', 'import a.b; x = 1', 'import a.b.; x = 1', @@ -1222,6 +1237,8 @@ def test_parse_error(self): 'from foo import import', 'from foo import from', 'from foo import as', + 'from \\x', # _tokenize SyntaxError -> tokenize TokenError + 'if 1:\n pass\n\tpass', # _tokenize TabError -> tokenize TabError ) for code in cases: parser = ImportParser(code) From 713684de5311eb9edb47f2f5fe3f4160f8d35e5a Mon Sep 17 00:00:00 2001 From: "Tomas R." Date: Sun, 28 Dec 2025 22:06:06 +0100 Subject: [PATCH 603/638] gh-131798: Remove bounds check when indexing into tuples with a constant index (#137607) * Remove bounds check when indexing into tuples with a constant index * Add news entry * fixup after rebase --- Include/internal/pycore_opcode_metadata.h | 2 +- Include/internal/pycore_uop_ids.h | 1859 +++++++++-------- Include/internal/pycore_uop_metadata.h | 33 +- Lib/test/test_capi/test_opt.py | 15 + ...-08-10-12-46-36.gh-issue-131798.5ys0H_.rst | 1 + Python/bytecodes.c | 21 +- Python/executor_cases.c.h | 196 +- Python/generated_cases.c.h | 12 +- Python/optimizer_bytecodes.c | 13 + Python/optimizer_cases.c.h | 18 + 10 files changed, 1231 insertions(+), 939 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-08-10-12-46-36.gh-issue-131798.5ys0H_.rst diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index cd2475cd2374e8..424ec337eb8afd 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1341,7 +1341,7 @@ _PyOpcode_macro_expansion[256] = { [BINARY_OP_SUBSCR_LIST_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_LIST_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 } } }, [BINARY_OP_SUBSCR_LIST_SLICE] = { .nuops = 3, .uops = { { _GUARD_TOS_SLICE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_LIST_SLICE, OPARG_SIMPLE, 5 } } }, [BINARY_OP_SUBSCR_STR_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_UNICODE, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_STR_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_SUBSCR_TUPLE_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_TUPLE, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_TUPLE_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_SUBSCR_TUPLE_INT] = { .nuops = 6, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_TUPLE, OPARG_SIMPLE, 0 }, { _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_TUPLE_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 } } }, [BINARY_OP_SUBTRACT_FLOAT] = { .nuops = 5, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBTRACT_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 } } }, [BINARY_OP_SUBTRACT_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBTRACT_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 } } }, [BINARY_SLICE] = { .nuops = 1, .uops = { { _BINARY_SLICE, OPARG_SIMPLE, 0 } } }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index b146c4ea39b4a9..ebeec6387a741a 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -126,59 +126,60 @@ extern "C" { #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER #define _GUARD_BINARY_OP_EXTEND 383 -#define _GUARD_CALLABLE_ISINSTANCE 384 -#define _GUARD_CALLABLE_LEN 385 -#define _GUARD_CALLABLE_LIST_APPEND 386 -#define _GUARD_CALLABLE_STR_1 387 -#define _GUARD_CALLABLE_TUPLE_1 388 -#define _GUARD_CALLABLE_TYPE_1 389 -#define _GUARD_DORV_NO_DICT 390 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 391 -#define _GUARD_GLOBALS_VERSION 392 -#define _GUARD_IP_RETURN_GENERATOR 393 -#define _GUARD_IP_RETURN_VALUE 394 -#define _GUARD_IP_YIELD_VALUE 395 -#define _GUARD_IP__PUSH_FRAME 396 -#define _GUARD_IS_FALSE_POP 397 -#define _GUARD_IS_NONE_POP 398 -#define _GUARD_IS_NOT_NONE_POP 399 -#define _GUARD_IS_TRUE_POP 400 -#define _GUARD_KEYS_VERSION 401 -#define _GUARD_NOS_DICT 402 -#define _GUARD_NOS_FLOAT 403 -#define _GUARD_NOS_INT 404 -#define _GUARD_NOS_LIST 405 -#define _GUARD_NOS_NOT_NULL 406 -#define _GUARD_NOS_NULL 407 -#define _GUARD_NOS_OVERFLOWED 408 -#define _GUARD_NOS_TUPLE 409 -#define _GUARD_NOS_UNICODE 410 -#define _GUARD_NOT_EXHAUSTED_LIST 411 -#define _GUARD_NOT_EXHAUSTED_RANGE 412 -#define _GUARD_NOT_EXHAUSTED_TUPLE 413 -#define _GUARD_THIRD_NULL 414 -#define _GUARD_TOS_ANY_SET 415 -#define _GUARD_TOS_DICT 416 -#define _GUARD_TOS_FLOAT 417 -#define _GUARD_TOS_INT 418 -#define _GUARD_TOS_LIST 419 -#define _GUARD_TOS_OVERFLOWED 420 -#define _GUARD_TOS_SLICE 421 -#define _GUARD_TOS_TUPLE 422 -#define _GUARD_TOS_UNICODE 423 -#define _GUARD_TYPE_VERSION 424 -#define _GUARD_TYPE_VERSION_AND_LOCK 425 -#define _HANDLE_PENDING_AND_DEOPT 426 +#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS 384 +#define _GUARD_CALLABLE_ISINSTANCE 385 +#define _GUARD_CALLABLE_LEN 386 +#define _GUARD_CALLABLE_LIST_APPEND 387 +#define _GUARD_CALLABLE_STR_1 388 +#define _GUARD_CALLABLE_TUPLE_1 389 +#define _GUARD_CALLABLE_TYPE_1 390 +#define _GUARD_DORV_NO_DICT 391 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 392 +#define _GUARD_GLOBALS_VERSION 393 +#define _GUARD_IP_RETURN_GENERATOR 394 +#define _GUARD_IP_RETURN_VALUE 395 +#define _GUARD_IP_YIELD_VALUE 396 +#define _GUARD_IP__PUSH_FRAME 397 +#define _GUARD_IS_FALSE_POP 398 +#define _GUARD_IS_NONE_POP 399 +#define _GUARD_IS_NOT_NONE_POP 400 +#define _GUARD_IS_TRUE_POP 401 +#define _GUARD_KEYS_VERSION 402 +#define _GUARD_NOS_DICT 403 +#define _GUARD_NOS_FLOAT 404 +#define _GUARD_NOS_INT 405 +#define _GUARD_NOS_LIST 406 +#define _GUARD_NOS_NOT_NULL 407 +#define _GUARD_NOS_NULL 408 +#define _GUARD_NOS_OVERFLOWED 409 +#define _GUARD_NOS_TUPLE 410 +#define _GUARD_NOS_UNICODE 411 +#define _GUARD_NOT_EXHAUSTED_LIST 412 +#define _GUARD_NOT_EXHAUSTED_RANGE 413 +#define _GUARD_NOT_EXHAUSTED_TUPLE 414 +#define _GUARD_THIRD_NULL 415 +#define _GUARD_TOS_ANY_SET 416 +#define _GUARD_TOS_DICT 417 +#define _GUARD_TOS_FLOAT 418 +#define _GUARD_TOS_INT 419 +#define _GUARD_TOS_LIST 420 +#define _GUARD_TOS_OVERFLOWED 421 +#define _GUARD_TOS_SLICE 422 +#define _GUARD_TOS_TUPLE 423 +#define _GUARD_TOS_UNICODE 424 +#define _GUARD_TYPE_VERSION 425 +#define _GUARD_TYPE_VERSION_AND_LOCK 426 +#define _HANDLE_PENDING_AND_DEOPT 427 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 427 -#define _INIT_CALL_PY_EXACT_ARGS 428 -#define _INIT_CALL_PY_EXACT_ARGS_0 429 -#define _INIT_CALL_PY_EXACT_ARGS_1 430 -#define _INIT_CALL_PY_EXACT_ARGS_2 431 -#define _INIT_CALL_PY_EXACT_ARGS_3 432 -#define _INIT_CALL_PY_EXACT_ARGS_4 433 -#define _INSERT_NULL 434 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 428 +#define _INIT_CALL_PY_EXACT_ARGS 429 +#define _INIT_CALL_PY_EXACT_ARGS_0 430 +#define _INIT_CALL_PY_EXACT_ARGS_1 431 +#define _INIT_CALL_PY_EXACT_ARGS_2 432 +#define _INIT_CALL_PY_EXACT_ARGS_3 433 +#define _INIT_CALL_PY_EXACT_ARGS_4 434 +#define _INSERT_NULL 435 #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER #define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD @@ -188,930 +189,936 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _IS_NONE 435 -#define _IS_OP 436 -#define _ITER_CHECK_LIST 437 -#define _ITER_CHECK_RANGE 438 -#define _ITER_CHECK_TUPLE 439 -#define _ITER_JUMP_LIST 440 -#define _ITER_JUMP_RANGE 441 -#define _ITER_JUMP_TUPLE 442 -#define _ITER_NEXT_LIST 443 -#define _ITER_NEXT_LIST_TIER_TWO 444 -#define _ITER_NEXT_RANGE 445 -#define _ITER_NEXT_TUPLE 446 +#define _IS_NONE 436 +#define _IS_OP 437 +#define _ITER_CHECK_LIST 438 +#define _ITER_CHECK_RANGE 439 +#define _ITER_CHECK_TUPLE 440 +#define _ITER_JUMP_LIST 441 +#define _ITER_JUMP_RANGE 442 +#define _ITER_JUMP_TUPLE 443 +#define _ITER_NEXT_LIST 444 +#define _ITER_NEXT_LIST_TIER_TWO 445 +#define _ITER_NEXT_RANGE 446 +#define _ITER_NEXT_TUPLE 447 #define _JUMP_BACKWARD_NO_INTERRUPT JUMP_BACKWARD_NO_INTERRUPT -#define _JUMP_TO_TOP 447 +#define _JUMP_TO_TOP 448 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 448 -#define _LOAD_ATTR_CLASS 449 +#define _LOAD_ATTR 449 +#define _LOAD_ATTR_CLASS 450 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 450 -#define _LOAD_ATTR_METHOD_LAZY_DICT 451 -#define _LOAD_ATTR_METHOD_NO_DICT 452 -#define _LOAD_ATTR_METHOD_WITH_VALUES 453 -#define _LOAD_ATTR_MODULE 454 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 455 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 456 -#define _LOAD_ATTR_PROPERTY_FRAME 457 -#define _LOAD_ATTR_SLOT 458 -#define _LOAD_ATTR_WITH_HINT 459 +#define _LOAD_ATTR_INSTANCE_VALUE 451 +#define _LOAD_ATTR_METHOD_LAZY_DICT 452 +#define _LOAD_ATTR_METHOD_NO_DICT 453 +#define _LOAD_ATTR_METHOD_WITH_VALUES 454 +#define _LOAD_ATTR_MODULE 455 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 456 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 457 +#define _LOAD_ATTR_PROPERTY_FRAME 458 +#define _LOAD_ATTR_SLOT 459 +#define _LOAD_ATTR_WITH_HINT 460 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 460 +#define _LOAD_BYTECODE 461 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 461 -#define _LOAD_CONST_INLINE_BORROW 462 -#define _LOAD_CONST_UNDER_INLINE 463 -#define _LOAD_CONST_UNDER_INLINE_BORROW 464 +#define _LOAD_CONST_INLINE 462 +#define _LOAD_CONST_INLINE_BORROW 463 +#define _LOAD_CONST_UNDER_INLINE 464 +#define _LOAD_CONST_UNDER_INLINE_BORROW 465 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 465 -#define _LOAD_FAST_0 466 -#define _LOAD_FAST_1 467 -#define _LOAD_FAST_2 468 -#define _LOAD_FAST_3 469 -#define _LOAD_FAST_4 470 -#define _LOAD_FAST_5 471 -#define _LOAD_FAST_6 472 -#define _LOAD_FAST_7 473 +#define _LOAD_FAST 466 +#define _LOAD_FAST_0 467 +#define _LOAD_FAST_1 468 +#define _LOAD_FAST_2 469 +#define _LOAD_FAST_3 470 +#define _LOAD_FAST_4 471 +#define _LOAD_FAST_5 472 +#define _LOAD_FAST_6 473 +#define _LOAD_FAST_7 474 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_BORROW 474 -#define _LOAD_FAST_BORROW_0 475 -#define _LOAD_FAST_BORROW_1 476 -#define _LOAD_FAST_BORROW_2 477 -#define _LOAD_FAST_BORROW_3 478 -#define _LOAD_FAST_BORROW_4 479 -#define _LOAD_FAST_BORROW_5 480 -#define _LOAD_FAST_BORROW_6 481 -#define _LOAD_FAST_BORROW_7 482 +#define _LOAD_FAST_BORROW 475 +#define _LOAD_FAST_BORROW_0 476 +#define _LOAD_FAST_BORROW_1 477 +#define _LOAD_FAST_BORROW_2 478 +#define _LOAD_FAST_BORROW_3 479 +#define _LOAD_FAST_BORROW_4 480 +#define _LOAD_FAST_BORROW_5 481 +#define _LOAD_FAST_BORROW_6 482 +#define _LOAD_FAST_BORROW_7 483 #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 483 -#define _LOAD_GLOBAL_BUILTINS 484 -#define _LOAD_GLOBAL_MODULE 485 +#define _LOAD_GLOBAL 484 +#define _LOAD_GLOBAL_BUILTINS 485 +#define _LOAD_GLOBAL_MODULE 486 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 486 -#define _LOAD_SMALL_INT_0 487 -#define _LOAD_SMALL_INT_1 488 -#define _LOAD_SMALL_INT_2 489 -#define _LOAD_SMALL_INT_3 490 -#define _LOAD_SPECIAL 491 +#define _LOAD_SMALL_INT 487 +#define _LOAD_SMALL_INT_0 488 +#define _LOAD_SMALL_INT_1 489 +#define _LOAD_SMALL_INT_2 490 +#define _LOAD_SMALL_INT_3 491 +#define _LOAD_SPECIAL 492 #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 492 +#define _MAKE_CALLARGS_A_TUPLE 493 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 493 +#define _MAKE_WARM 494 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 494 -#define _MAYBE_EXPAND_METHOD_KW 495 -#define _MONITOR_CALL 496 -#define _MONITOR_CALL_KW 497 -#define _MONITOR_JUMP_BACKWARD 498 -#define _MONITOR_RESUME 499 +#define _MAYBE_EXPAND_METHOD 495 +#define _MAYBE_EXPAND_METHOD_KW 496 +#define _MONITOR_CALL 497 +#define _MONITOR_CALL_KW 498 +#define _MONITOR_JUMP_BACKWARD 499 +#define _MONITOR_RESUME 500 #define _NOP NOP -#define _POP_CALL 500 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW 501 -#define _POP_CALL_ONE 502 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 503 -#define _POP_CALL_TWO 504 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 505 +#define _POP_CALL 501 +#define _POP_CALL_LOAD_CONST_INLINE_BORROW 502 +#define _POP_CALL_ONE 503 +#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 504 +#define _POP_CALL_TWO 505 +#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 506 #define _POP_EXCEPT POP_EXCEPT #define _POP_ITER POP_ITER -#define _POP_JUMP_IF_FALSE 506 -#define _POP_JUMP_IF_TRUE 507 +#define _POP_JUMP_IF_FALSE 507 +#define _POP_JUMP_IF_TRUE 508 #define _POP_TOP POP_TOP -#define _POP_TOP_FLOAT 508 -#define _POP_TOP_INT 509 -#define _POP_TOP_LOAD_CONST_INLINE 510 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 511 -#define _POP_TOP_NOP 512 -#define _POP_TOP_UNICODE 513 -#define _POP_TWO 514 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW 515 +#define _POP_TOP_FLOAT 509 +#define _POP_TOP_INT 510 +#define _POP_TOP_LOAD_CONST_INLINE 511 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 512 +#define _POP_TOP_NOP 513 +#define _POP_TOP_UNICODE 514 +#define _POP_TWO 515 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW 516 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 516 +#define _PUSH_FRAME 517 #define _PUSH_NULL PUSH_NULL -#define _PUSH_NULL_CONDITIONAL 517 -#define _PY_FRAME_GENERAL 518 -#define _PY_FRAME_KW 519 -#define _QUICKEN_RESUME 520 -#define _REPLACE_WITH_TRUE 521 +#define _PUSH_NULL_CONDITIONAL 518 +#define _PY_FRAME_GENERAL 519 +#define _PY_FRAME_KW 520 +#define _QUICKEN_RESUME 521 +#define _REPLACE_WITH_TRUE 522 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 522 -#define _SEND 523 -#define _SEND_GEN_FRAME 524 +#define _SAVE_RETURN_OFFSET 523 +#define _SEND 524 +#define _SEND_GEN_FRAME 525 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW 525 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW 526 -#define _SPILL_OR_RELOAD 527 -#define _START_EXECUTOR 528 -#define _STORE_ATTR 529 -#define _STORE_ATTR_INSTANCE_VALUE 530 -#define _STORE_ATTR_SLOT 531 -#define _STORE_ATTR_WITH_HINT 532 +#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW 526 +#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW 527 +#define _SPILL_OR_RELOAD 528 +#define _START_EXECUTOR 529 +#define _STORE_ATTR 530 +#define _STORE_ATTR_INSTANCE_VALUE 531 +#define _STORE_ATTR_SLOT 532 +#define _STORE_ATTR_WITH_HINT 533 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 533 -#define _STORE_FAST_0 534 -#define _STORE_FAST_1 535 -#define _STORE_FAST_2 536 -#define _STORE_FAST_3 537 -#define _STORE_FAST_4 538 -#define _STORE_FAST_5 539 -#define _STORE_FAST_6 540 -#define _STORE_FAST_7 541 +#define _STORE_FAST 534 +#define _STORE_FAST_0 535 +#define _STORE_FAST_1 536 +#define _STORE_FAST_2 537 +#define _STORE_FAST_3 538 +#define _STORE_FAST_4 539 +#define _STORE_FAST_5 540 +#define _STORE_FAST_6 541 +#define _STORE_FAST_7 542 #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 542 -#define _STORE_SUBSCR 543 -#define _STORE_SUBSCR_DICT 544 -#define _STORE_SUBSCR_LIST_INT 545 -#define _SWAP 546 -#define _SWAP_2 547 -#define _SWAP_3 548 -#define _TIER2_RESUME_CHECK 549 -#define _TO_BOOL 550 +#define _STORE_SLICE 543 +#define _STORE_SUBSCR 544 +#define _STORE_SUBSCR_DICT 545 +#define _STORE_SUBSCR_LIST_INT 546 +#define _SWAP 547 +#define _SWAP_2 548 +#define _SWAP_3 549 +#define _TIER2_RESUME_CHECK 550 +#define _TO_BOOL 551 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT -#define _TO_BOOL_LIST 551 +#define _TO_BOOL_LIST 552 #define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR 552 +#define _TO_BOOL_STR 553 #define _TRACE_RECORD TRACE_RECORD #define _UNARY_INVERT UNARY_INVERT #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 553 -#define _UNPACK_SEQUENCE_LIST 554 -#define _UNPACK_SEQUENCE_TUPLE 555 -#define _UNPACK_SEQUENCE_TWO_TUPLE 556 +#define _UNPACK_SEQUENCE 554 +#define _UNPACK_SEQUENCE_LIST 555 +#define _UNPACK_SEQUENCE_TUPLE 556 +#define _UNPACK_SEQUENCE_TWO_TUPLE 557 #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 556 -#define _BINARY_OP_r21 557 -#define _BINARY_OP_ADD_FLOAT_r03 558 -#define _BINARY_OP_ADD_FLOAT_r13 559 -#define _BINARY_OP_ADD_FLOAT_r23 560 -#define _BINARY_OP_ADD_INT_r03 561 -#define _BINARY_OP_ADD_INT_r13 562 -#define _BINARY_OP_ADD_INT_r23 563 -#define _BINARY_OP_ADD_UNICODE_r03 564 -#define _BINARY_OP_ADD_UNICODE_r13 565 -#define _BINARY_OP_ADD_UNICODE_r23 566 -#define _BINARY_OP_EXTEND_r21 567 -#define _BINARY_OP_INPLACE_ADD_UNICODE_r21 568 -#define _BINARY_OP_MULTIPLY_FLOAT_r03 569 -#define _BINARY_OP_MULTIPLY_FLOAT_r13 570 -#define _BINARY_OP_MULTIPLY_FLOAT_r23 571 -#define _BINARY_OP_MULTIPLY_INT_r03 572 -#define _BINARY_OP_MULTIPLY_INT_r13 573 -#define _BINARY_OP_MULTIPLY_INT_r23 574 -#define _BINARY_OP_SUBSCR_CHECK_FUNC_r23 575 -#define _BINARY_OP_SUBSCR_DICT_r21 576 -#define _BINARY_OP_SUBSCR_INIT_CALL_r01 577 -#define _BINARY_OP_SUBSCR_INIT_CALL_r11 578 -#define _BINARY_OP_SUBSCR_INIT_CALL_r21 579 -#define _BINARY_OP_SUBSCR_INIT_CALL_r31 580 -#define _BINARY_OP_SUBSCR_LIST_INT_r23 581 -#define _BINARY_OP_SUBSCR_LIST_SLICE_r21 582 -#define _BINARY_OP_SUBSCR_STR_INT_r23 583 -#define _BINARY_OP_SUBSCR_TUPLE_INT_r23 584 -#define _BINARY_OP_SUBTRACT_FLOAT_r03 585 -#define _BINARY_OP_SUBTRACT_FLOAT_r13 586 -#define _BINARY_OP_SUBTRACT_FLOAT_r23 587 -#define _BINARY_OP_SUBTRACT_INT_r03 588 -#define _BINARY_OP_SUBTRACT_INT_r13 589 -#define _BINARY_OP_SUBTRACT_INT_r23 590 -#define _BINARY_SLICE_r31 591 -#define _BUILD_INTERPOLATION_r01 592 -#define _BUILD_LIST_r01 593 -#define _BUILD_MAP_r01 594 -#define _BUILD_SET_r01 595 -#define _BUILD_SLICE_r01 596 -#define _BUILD_STRING_r01 597 -#define _BUILD_TEMPLATE_r21 598 -#define _BUILD_TUPLE_r01 599 -#define _CALL_BUILTIN_CLASS_r01 600 -#define _CALL_BUILTIN_FAST_r01 601 -#define _CALL_BUILTIN_FAST_WITH_KEYWORDS_r01 602 -#define _CALL_BUILTIN_O_r03 603 -#define _CALL_INTRINSIC_1_r11 604 -#define _CALL_INTRINSIC_2_r21 605 -#define _CALL_ISINSTANCE_r31 606 -#define _CALL_KW_NON_PY_r11 607 -#define _CALL_LEN_r33 608 -#define _CALL_LIST_APPEND_r03 609 -#define _CALL_LIST_APPEND_r13 610 -#define _CALL_LIST_APPEND_r23 611 -#define _CALL_LIST_APPEND_r33 612 -#define _CALL_METHOD_DESCRIPTOR_FAST_r01 613 -#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r01 614 -#define _CALL_METHOD_DESCRIPTOR_NOARGS_r01 615 -#define _CALL_METHOD_DESCRIPTOR_O_r01 616 -#define _CALL_NON_PY_GENERAL_r01 617 -#define _CALL_STR_1_r32 618 -#define _CALL_TUPLE_1_r32 619 -#define _CALL_TYPE_1_r02 620 -#define _CALL_TYPE_1_r12 621 -#define _CALL_TYPE_1_r22 622 -#define _CALL_TYPE_1_r32 623 -#define _CHECK_AND_ALLOCATE_OBJECT_r00 624 -#define _CHECK_ATTR_CLASS_r01 625 -#define _CHECK_ATTR_CLASS_r11 626 -#define _CHECK_ATTR_CLASS_r22 627 -#define _CHECK_ATTR_CLASS_r33 628 -#define _CHECK_ATTR_METHOD_LAZY_DICT_r01 629 -#define _CHECK_ATTR_METHOD_LAZY_DICT_r11 630 -#define _CHECK_ATTR_METHOD_LAZY_DICT_r22 631 -#define _CHECK_ATTR_METHOD_LAZY_DICT_r33 632 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00 633 -#define _CHECK_EG_MATCH_r22 634 -#define _CHECK_EXC_MATCH_r22 635 -#define _CHECK_FUNCTION_EXACT_ARGS_r00 636 -#define _CHECK_FUNCTION_VERSION_r00 637 -#define _CHECK_FUNCTION_VERSION_INLINE_r00 638 -#define _CHECK_FUNCTION_VERSION_INLINE_r11 639 -#define _CHECK_FUNCTION_VERSION_INLINE_r22 640 -#define _CHECK_FUNCTION_VERSION_INLINE_r33 641 -#define _CHECK_FUNCTION_VERSION_KW_r11 642 -#define _CHECK_IS_NOT_PY_CALLABLE_r00 643 -#define _CHECK_IS_NOT_PY_CALLABLE_KW_r11 644 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r01 645 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r11 646 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r22 647 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r33 648 -#define _CHECK_METHOD_VERSION_r00 649 -#define _CHECK_METHOD_VERSION_KW_r11 650 -#define _CHECK_PEP_523_r00 651 -#define _CHECK_PEP_523_r11 652 -#define _CHECK_PEP_523_r22 653 -#define _CHECK_PEP_523_r33 654 -#define _CHECK_PERIODIC_r00 655 -#define _CHECK_PERIODIC_AT_END_r00 656 -#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00 657 -#define _CHECK_RECURSION_REMAINING_r00 658 -#define _CHECK_RECURSION_REMAINING_r11 659 -#define _CHECK_RECURSION_REMAINING_r22 660 -#define _CHECK_RECURSION_REMAINING_r33 661 -#define _CHECK_STACK_SPACE_r00 662 -#define _CHECK_STACK_SPACE_OPERAND_r00 663 -#define _CHECK_STACK_SPACE_OPERAND_r11 664 -#define _CHECK_STACK_SPACE_OPERAND_r22 665 -#define _CHECK_STACK_SPACE_OPERAND_r33 666 -#define _CHECK_VALIDITY_r00 667 -#define _CHECK_VALIDITY_r11 668 -#define _CHECK_VALIDITY_r22 669 -#define _CHECK_VALIDITY_r33 670 -#define _COLD_DYNAMIC_EXIT_r00 671 -#define _COLD_EXIT_r00 672 -#define _COMPARE_OP_r21 673 -#define _COMPARE_OP_FLOAT_r03 674 -#define _COMPARE_OP_FLOAT_r13 675 -#define _COMPARE_OP_FLOAT_r23 676 -#define _COMPARE_OP_INT_r23 677 -#define _COMPARE_OP_STR_r23 678 -#define _CONTAINS_OP_r21 679 -#define _CONTAINS_OP_DICT_r21 680 -#define _CONTAINS_OP_SET_r21 681 -#define _CONVERT_VALUE_r11 682 -#define _COPY_r01 683 -#define _COPY_1_r02 684 -#define _COPY_1_r12 685 -#define _COPY_1_r23 686 -#define _COPY_2_r03 687 -#define _COPY_2_r13 688 -#define _COPY_2_r23 689 -#define _COPY_3_r03 690 -#define _COPY_3_r13 691 -#define _COPY_3_r23 692 -#define _COPY_3_r33 693 -#define _COPY_FREE_VARS_r00 694 -#define _COPY_FREE_VARS_r11 695 -#define _COPY_FREE_VARS_r22 696 -#define _COPY_FREE_VARS_r33 697 -#define _CREATE_INIT_FRAME_r01 698 -#define _DELETE_ATTR_r10 699 -#define _DELETE_DEREF_r00 700 -#define _DELETE_FAST_r00 701 -#define _DELETE_GLOBAL_r00 702 -#define _DELETE_NAME_r00 703 -#define _DELETE_SUBSCR_r20 704 -#define _DEOPT_r00 705 -#define _DEOPT_r10 706 -#define _DEOPT_r20 707 -#define _DEOPT_r30 708 -#define _DICT_MERGE_r10 709 -#define _DICT_UPDATE_r10 710 -#define _DO_CALL_r01 711 -#define _DO_CALL_FUNCTION_EX_r31 712 -#define _DO_CALL_KW_r11 713 -#define _DYNAMIC_EXIT_r00 714 -#define _DYNAMIC_EXIT_r10 715 -#define _DYNAMIC_EXIT_r20 716 -#define _DYNAMIC_EXIT_r30 717 -#define _END_FOR_r10 718 -#define _END_SEND_r21 719 -#define _ERROR_POP_N_r00 720 -#define _EXIT_INIT_CHECK_r10 721 -#define _EXIT_TRACE_r00 722 -#define _EXIT_TRACE_r10 723 -#define _EXIT_TRACE_r20 724 -#define _EXIT_TRACE_r30 725 -#define _EXPAND_METHOD_r00 726 -#define _EXPAND_METHOD_KW_r11 727 -#define _FATAL_ERROR_r00 728 -#define _FATAL_ERROR_r11 729 -#define _FATAL_ERROR_r22 730 -#define _FATAL_ERROR_r33 731 -#define _FORMAT_SIMPLE_r11 732 -#define _FORMAT_WITH_SPEC_r21 733 -#define _FOR_ITER_r23 734 -#define _FOR_ITER_GEN_FRAME_r03 735 -#define _FOR_ITER_GEN_FRAME_r13 736 -#define _FOR_ITER_GEN_FRAME_r23 737 -#define _FOR_ITER_TIER_TWO_r23 738 -#define _GET_AITER_r11 739 -#define _GET_ANEXT_r12 740 -#define _GET_AWAITABLE_r11 741 -#define _GET_ITER_r12 742 -#define _GET_LEN_r12 743 -#define _GET_YIELD_FROM_ITER_r11 744 -#define _GUARD_BINARY_OP_EXTEND_r22 745 -#define _GUARD_CALLABLE_ISINSTANCE_r03 746 -#define _GUARD_CALLABLE_ISINSTANCE_r13 747 -#define _GUARD_CALLABLE_ISINSTANCE_r23 748 -#define _GUARD_CALLABLE_ISINSTANCE_r33 749 -#define _GUARD_CALLABLE_LEN_r03 750 -#define _GUARD_CALLABLE_LEN_r13 751 -#define _GUARD_CALLABLE_LEN_r23 752 -#define _GUARD_CALLABLE_LEN_r33 753 -#define _GUARD_CALLABLE_LIST_APPEND_r03 754 -#define _GUARD_CALLABLE_LIST_APPEND_r13 755 -#define _GUARD_CALLABLE_LIST_APPEND_r23 756 -#define _GUARD_CALLABLE_LIST_APPEND_r33 757 -#define _GUARD_CALLABLE_STR_1_r03 758 -#define _GUARD_CALLABLE_STR_1_r13 759 -#define _GUARD_CALLABLE_STR_1_r23 760 -#define _GUARD_CALLABLE_STR_1_r33 761 -#define _GUARD_CALLABLE_TUPLE_1_r03 762 -#define _GUARD_CALLABLE_TUPLE_1_r13 763 -#define _GUARD_CALLABLE_TUPLE_1_r23 764 -#define _GUARD_CALLABLE_TUPLE_1_r33 765 -#define _GUARD_CALLABLE_TYPE_1_r03 766 -#define _GUARD_CALLABLE_TYPE_1_r13 767 -#define _GUARD_CALLABLE_TYPE_1_r23 768 -#define _GUARD_CALLABLE_TYPE_1_r33 769 -#define _GUARD_DORV_NO_DICT_r01 770 -#define _GUARD_DORV_NO_DICT_r11 771 -#define _GUARD_DORV_NO_DICT_r22 772 -#define _GUARD_DORV_NO_DICT_r33 773 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r01 774 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11 775 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22 776 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33 777 -#define _GUARD_GLOBALS_VERSION_r00 778 -#define _GUARD_GLOBALS_VERSION_r11 779 -#define _GUARD_GLOBALS_VERSION_r22 780 -#define _GUARD_GLOBALS_VERSION_r33 781 -#define _GUARD_IP_RETURN_GENERATOR_r00 782 -#define _GUARD_IP_RETURN_GENERATOR_r11 783 -#define _GUARD_IP_RETURN_GENERATOR_r22 784 -#define _GUARD_IP_RETURN_GENERATOR_r33 785 -#define _GUARD_IP_RETURN_VALUE_r00 786 -#define _GUARD_IP_RETURN_VALUE_r11 787 -#define _GUARD_IP_RETURN_VALUE_r22 788 -#define _GUARD_IP_RETURN_VALUE_r33 789 -#define _GUARD_IP_YIELD_VALUE_r00 790 -#define _GUARD_IP_YIELD_VALUE_r11 791 -#define _GUARD_IP_YIELD_VALUE_r22 792 -#define _GUARD_IP_YIELD_VALUE_r33 793 -#define _GUARD_IP__PUSH_FRAME_r00 794 -#define _GUARD_IP__PUSH_FRAME_r11 795 -#define _GUARD_IP__PUSH_FRAME_r22 796 -#define _GUARD_IP__PUSH_FRAME_r33 797 -#define _GUARD_IS_FALSE_POP_r00 798 -#define _GUARD_IS_FALSE_POP_r10 799 -#define _GUARD_IS_FALSE_POP_r21 800 -#define _GUARD_IS_FALSE_POP_r32 801 -#define _GUARD_IS_NONE_POP_r00 802 -#define _GUARD_IS_NONE_POP_r10 803 -#define _GUARD_IS_NONE_POP_r21 804 -#define _GUARD_IS_NONE_POP_r32 805 -#define _GUARD_IS_NOT_NONE_POP_r10 806 -#define _GUARD_IS_TRUE_POP_r00 807 -#define _GUARD_IS_TRUE_POP_r10 808 -#define _GUARD_IS_TRUE_POP_r21 809 -#define _GUARD_IS_TRUE_POP_r32 810 -#define _GUARD_KEYS_VERSION_r01 811 -#define _GUARD_KEYS_VERSION_r11 812 -#define _GUARD_KEYS_VERSION_r22 813 -#define _GUARD_KEYS_VERSION_r33 814 -#define _GUARD_NOS_DICT_r02 815 -#define _GUARD_NOS_DICT_r12 816 -#define _GUARD_NOS_DICT_r22 817 -#define _GUARD_NOS_DICT_r33 818 -#define _GUARD_NOS_FLOAT_r02 819 -#define _GUARD_NOS_FLOAT_r12 820 -#define _GUARD_NOS_FLOAT_r22 821 -#define _GUARD_NOS_FLOAT_r33 822 -#define _GUARD_NOS_INT_r02 823 -#define _GUARD_NOS_INT_r12 824 -#define _GUARD_NOS_INT_r22 825 -#define _GUARD_NOS_INT_r33 826 -#define _GUARD_NOS_LIST_r02 827 -#define _GUARD_NOS_LIST_r12 828 -#define _GUARD_NOS_LIST_r22 829 -#define _GUARD_NOS_LIST_r33 830 -#define _GUARD_NOS_NOT_NULL_r02 831 -#define _GUARD_NOS_NOT_NULL_r12 832 -#define _GUARD_NOS_NOT_NULL_r22 833 -#define _GUARD_NOS_NOT_NULL_r33 834 -#define _GUARD_NOS_NULL_r02 835 -#define _GUARD_NOS_NULL_r12 836 -#define _GUARD_NOS_NULL_r22 837 -#define _GUARD_NOS_NULL_r33 838 -#define _GUARD_NOS_OVERFLOWED_r02 839 -#define _GUARD_NOS_OVERFLOWED_r12 840 -#define _GUARD_NOS_OVERFLOWED_r22 841 -#define _GUARD_NOS_OVERFLOWED_r33 842 -#define _GUARD_NOS_TUPLE_r02 843 -#define _GUARD_NOS_TUPLE_r12 844 -#define _GUARD_NOS_TUPLE_r22 845 -#define _GUARD_NOS_TUPLE_r33 846 -#define _GUARD_NOS_UNICODE_r02 847 -#define _GUARD_NOS_UNICODE_r12 848 -#define _GUARD_NOS_UNICODE_r22 849 -#define _GUARD_NOS_UNICODE_r33 850 -#define _GUARD_NOT_EXHAUSTED_LIST_r02 851 -#define _GUARD_NOT_EXHAUSTED_LIST_r12 852 -#define _GUARD_NOT_EXHAUSTED_LIST_r22 853 -#define _GUARD_NOT_EXHAUSTED_LIST_r33 854 -#define _GUARD_NOT_EXHAUSTED_RANGE_r02 855 -#define _GUARD_NOT_EXHAUSTED_RANGE_r12 856 -#define _GUARD_NOT_EXHAUSTED_RANGE_r22 857 -#define _GUARD_NOT_EXHAUSTED_RANGE_r33 858 -#define _GUARD_NOT_EXHAUSTED_TUPLE_r02 859 -#define _GUARD_NOT_EXHAUSTED_TUPLE_r12 860 -#define _GUARD_NOT_EXHAUSTED_TUPLE_r22 861 -#define _GUARD_NOT_EXHAUSTED_TUPLE_r33 862 -#define _GUARD_THIRD_NULL_r03 863 -#define _GUARD_THIRD_NULL_r13 864 -#define _GUARD_THIRD_NULL_r23 865 -#define _GUARD_THIRD_NULL_r33 866 -#define _GUARD_TOS_ANY_SET_r01 867 -#define _GUARD_TOS_ANY_SET_r11 868 -#define _GUARD_TOS_ANY_SET_r22 869 -#define _GUARD_TOS_ANY_SET_r33 870 -#define _GUARD_TOS_DICT_r01 871 -#define _GUARD_TOS_DICT_r11 872 -#define _GUARD_TOS_DICT_r22 873 -#define _GUARD_TOS_DICT_r33 874 -#define _GUARD_TOS_FLOAT_r01 875 -#define _GUARD_TOS_FLOAT_r11 876 -#define _GUARD_TOS_FLOAT_r22 877 -#define _GUARD_TOS_FLOAT_r33 878 -#define _GUARD_TOS_INT_r01 879 -#define _GUARD_TOS_INT_r11 880 -#define _GUARD_TOS_INT_r22 881 -#define _GUARD_TOS_INT_r33 882 -#define _GUARD_TOS_LIST_r01 883 -#define _GUARD_TOS_LIST_r11 884 -#define _GUARD_TOS_LIST_r22 885 -#define _GUARD_TOS_LIST_r33 886 -#define _GUARD_TOS_OVERFLOWED_r01 887 -#define _GUARD_TOS_OVERFLOWED_r11 888 -#define _GUARD_TOS_OVERFLOWED_r22 889 -#define _GUARD_TOS_OVERFLOWED_r33 890 -#define _GUARD_TOS_SLICE_r01 891 -#define _GUARD_TOS_SLICE_r11 892 -#define _GUARD_TOS_SLICE_r22 893 -#define _GUARD_TOS_SLICE_r33 894 -#define _GUARD_TOS_TUPLE_r01 895 -#define _GUARD_TOS_TUPLE_r11 896 -#define _GUARD_TOS_TUPLE_r22 897 -#define _GUARD_TOS_TUPLE_r33 898 -#define _GUARD_TOS_UNICODE_r01 899 -#define _GUARD_TOS_UNICODE_r11 900 -#define _GUARD_TOS_UNICODE_r22 901 -#define _GUARD_TOS_UNICODE_r33 902 -#define _GUARD_TYPE_VERSION_r01 903 -#define _GUARD_TYPE_VERSION_r11 904 -#define _GUARD_TYPE_VERSION_r22 905 -#define _GUARD_TYPE_VERSION_r33 906 -#define _GUARD_TYPE_VERSION_AND_LOCK_r01 907 -#define _GUARD_TYPE_VERSION_AND_LOCK_r11 908 -#define _GUARD_TYPE_VERSION_AND_LOCK_r22 909 -#define _GUARD_TYPE_VERSION_AND_LOCK_r33 910 -#define _HANDLE_PENDING_AND_DEOPT_r00 911 -#define _HANDLE_PENDING_AND_DEOPT_r10 912 -#define _HANDLE_PENDING_AND_DEOPT_r20 913 -#define _HANDLE_PENDING_AND_DEOPT_r30 914 -#define _IMPORT_FROM_r12 915 -#define _IMPORT_NAME_r21 916 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00 917 -#define _INIT_CALL_PY_EXACT_ARGS_r01 918 -#define _INIT_CALL_PY_EXACT_ARGS_0_r01 919 -#define _INIT_CALL_PY_EXACT_ARGS_1_r01 920 -#define _INIT_CALL_PY_EXACT_ARGS_2_r01 921 -#define _INIT_CALL_PY_EXACT_ARGS_3_r01 922 -#define _INIT_CALL_PY_EXACT_ARGS_4_r01 923 -#define _INSERT_NULL_r10 924 -#define _INSTRUMENTED_FOR_ITER_r23 925 -#define _INSTRUMENTED_INSTRUCTION_r00 926 -#define _INSTRUMENTED_JUMP_FORWARD_r00 927 -#define _INSTRUMENTED_JUMP_FORWARD_r11 928 -#define _INSTRUMENTED_JUMP_FORWARD_r22 929 -#define _INSTRUMENTED_JUMP_FORWARD_r33 930 -#define _INSTRUMENTED_LINE_r00 931 -#define _INSTRUMENTED_NOT_TAKEN_r00 932 -#define _INSTRUMENTED_NOT_TAKEN_r11 933 -#define _INSTRUMENTED_NOT_TAKEN_r22 934 -#define _INSTRUMENTED_NOT_TAKEN_r33 935 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r00 936 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r10 937 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r21 938 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r32 939 -#define _INSTRUMENTED_POP_JUMP_IF_NONE_r10 940 -#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE_r10 941 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r00 942 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r10 943 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r21 944 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r32 945 -#define _IS_NONE_r11 946 -#define _IS_OP_r03 947 -#define _IS_OP_r13 948 -#define _IS_OP_r23 949 -#define _ITER_CHECK_LIST_r02 950 -#define _ITER_CHECK_LIST_r12 951 -#define _ITER_CHECK_LIST_r22 952 -#define _ITER_CHECK_LIST_r33 953 -#define _ITER_CHECK_RANGE_r02 954 -#define _ITER_CHECK_RANGE_r12 955 -#define _ITER_CHECK_RANGE_r22 956 -#define _ITER_CHECK_RANGE_r33 957 -#define _ITER_CHECK_TUPLE_r02 958 -#define _ITER_CHECK_TUPLE_r12 959 -#define _ITER_CHECK_TUPLE_r22 960 -#define _ITER_CHECK_TUPLE_r33 961 -#define _ITER_JUMP_LIST_r02 962 -#define _ITER_JUMP_LIST_r12 963 -#define _ITER_JUMP_LIST_r22 964 -#define _ITER_JUMP_LIST_r33 965 -#define _ITER_JUMP_RANGE_r02 966 -#define _ITER_JUMP_RANGE_r12 967 -#define _ITER_JUMP_RANGE_r22 968 -#define _ITER_JUMP_RANGE_r33 969 -#define _ITER_JUMP_TUPLE_r02 970 -#define _ITER_JUMP_TUPLE_r12 971 -#define _ITER_JUMP_TUPLE_r22 972 -#define _ITER_JUMP_TUPLE_r33 973 -#define _ITER_NEXT_LIST_r23 974 -#define _ITER_NEXT_LIST_TIER_TWO_r23 975 -#define _ITER_NEXT_RANGE_r03 976 -#define _ITER_NEXT_RANGE_r13 977 -#define _ITER_NEXT_RANGE_r23 978 -#define _ITER_NEXT_TUPLE_r03 979 -#define _ITER_NEXT_TUPLE_r13 980 -#define _ITER_NEXT_TUPLE_r23 981 -#define _JUMP_BACKWARD_NO_INTERRUPT_r00 982 -#define _JUMP_BACKWARD_NO_INTERRUPT_r11 983 -#define _JUMP_BACKWARD_NO_INTERRUPT_r22 984 -#define _JUMP_BACKWARD_NO_INTERRUPT_r33 985 -#define _JUMP_TO_TOP_r00 986 -#define _LIST_APPEND_r10 987 -#define _LIST_EXTEND_r10 988 -#define _LOAD_ATTR_r10 989 -#define _LOAD_ATTR_CLASS_r11 990 -#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_r11 991 -#define _LOAD_ATTR_INSTANCE_VALUE_r02 992 -#define _LOAD_ATTR_INSTANCE_VALUE_r12 993 -#define _LOAD_ATTR_INSTANCE_VALUE_r23 994 -#define _LOAD_ATTR_METHOD_LAZY_DICT_r02 995 -#define _LOAD_ATTR_METHOD_LAZY_DICT_r12 996 -#define _LOAD_ATTR_METHOD_LAZY_DICT_r23 997 -#define _LOAD_ATTR_METHOD_NO_DICT_r02 998 -#define _LOAD_ATTR_METHOD_NO_DICT_r12 999 -#define _LOAD_ATTR_METHOD_NO_DICT_r23 1000 -#define _LOAD_ATTR_METHOD_WITH_VALUES_r02 1001 -#define _LOAD_ATTR_METHOD_WITH_VALUES_r12 1002 -#define _LOAD_ATTR_METHOD_WITH_VALUES_r23 1003 -#define _LOAD_ATTR_MODULE_r11 1004 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11 1005 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11 1006 -#define _LOAD_ATTR_PROPERTY_FRAME_r11 1007 -#define _LOAD_ATTR_SLOT_r11 1008 -#define _LOAD_ATTR_WITH_HINT_r12 1009 -#define _LOAD_BUILD_CLASS_r01 1010 -#define _LOAD_BYTECODE_r00 1011 -#define _LOAD_COMMON_CONSTANT_r01 1012 -#define _LOAD_COMMON_CONSTANT_r12 1013 -#define _LOAD_COMMON_CONSTANT_r23 1014 -#define _LOAD_CONST_r01 1015 -#define _LOAD_CONST_r12 1016 -#define _LOAD_CONST_r23 1017 -#define _LOAD_CONST_INLINE_r01 1018 -#define _LOAD_CONST_INLINE_r12 1019 -#define _LOAD_CONST_INLINE_r23 1020 -#define _LOAD_CONST_INLINE_BORROW_r01 1021 -#define _LOAD_CONST_INLINE_BORROW_r12 1022 -#define _LOAD_CONST_INLINE_BORROW_r23 1023 -#define _LOAD_CONST_UNDER_INLINE_r02 1024 -#define _LOAD_CONST_UNDER_INLINE_r12 1025 -#define _LOAD_CONST_UNDER_INLINE_r23 1026 -#define _LOAD_CONST_UNDER_INLINE_BORROW_r02 1027 -#define _LOAD_CONST_UNDER_INLINE_BORROW_r12 1028 -#define _LOAD_CONST_UNDER_INLINE_BORROW_r23 1029 -#define _LOAD_DEREF_r01 1030 -#define _LOAD_FAST_r01 1031 -#define _LOAD_FAST_r12 1032 -#define _LOAD_FAST_r23 1033 -#define _LOAD_FAST_0_r01 1034 -#define _LOAD_FAST_0_r12 1035 -#define _LOAD_FAST_0_r23 1036 -#define _LOAD_FAST_1_r01 1037 -#define _LOAD_FAST_1_r12 1038 -#define _LOAD_FAST_1_r23 1039 -#define _LOAD_FAST_2_r01 1040 -#define _LOAD_FAST_2_r12 1041 -#define _LOAD_FAST_2_r23 1042 -#define _LOAD_FAST_3_r01 1043 -#define _LOAD_FAST_3_r12 1044 -#define _LOAD_FAST_3_r23 1045 -#define _LOAD_FAST_4_r01 1046 -#define _LOAD_FAST_4_r12 1047 -#define _LOAD_FAST_4_r23 1048 -#define _LOAD_FAST_5_r01 1049 -#define _LOAD_FAST_5_r12 1050 -#define _LOAD_FAST_5_r23 1051 -#define _LOAD_FAST_6_r01 1052 -#define _LOAD_FAST_6_r12 1053 -#define _LOAD_FAST_6_r23 1054 -#define _LOAD_FAST_7_r01 1055 -#define _LOAD_FAST_7_r12 1056 -#define _LOAD_FAST_7_r23 1057 -#define _LOAD_FAST_AND_CLEAR_r01 1058 -#define _LOAD_FAST_AND_CLEAR_r12 1059 -#define _LOAD_FAST_AND_CLEAR_r23 1060 -#define _LOAD_FAST_BORROW_r01 1061 -#define _LOAD_FAST_BORROW_r12 1062 -#define _LOAD_FAST_BORROW_r23 1063 -#define _LOAD_FAST_BORROW_0_r01 1064 -#define _LOAD_FAST_BORROW_0_r12 1065 -#define _LOAD_FAST_BORROW_0_r23 1066 -#define _LOAD_FAST_BORROW_1_r01 1067 -#define _LOAD_FAST_BORROW_1_r12 1068 -#define _LOAD_FAST_BORROW_1_r23 1069 -#define _LOAD_FAST_BORROW_2_r01 1070 -#define _LOAD_FAST_BORROW_2_r12 1071 -#define _LOAD_FAST_BORROW_2_r23 1072 -#define _LOAD_FAST_BORROW_3_r01 1073 -#define _LOAD_FAST_BORROW_3_r12 1074 -#define _LOAD_FAST_BORROW_3_r23 1075 -#define _LOAD_FAST_BORROW_4_r01 1076 -#define _LOAD_FAST_BORROW_4_r12 1077 -#define _LOAD_FAST_BORROW_4_r23 1078 -#define _LOAD_FAST_BORROW_5_r01 1079 -#define _LOAD_FAST_BORROW_5_r12 1080 -#define _LOAD_FAST_BORROW_5_r23 1081 -#define _LOAD_FAST_BORROW_6_r01 1082 -#define _LOAD_FAST_BORROW_6_r12 1083 -#define _LOAD_FAST_BORROW_6_r23 1084 -#define _LOAD_FAST_BORROW_7_r01 1085 -#define _LOAD_FAST_BORROW_7_r12 1086 -#define _LOAD_FAST_BORROW_7_r23 1087 -#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r02 1088 -#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r13 1089 -#define _LOAD_FAST_CHECK_r01 1090 -#define _LOAD_FAST_CHECK_r12 1091 -#define _LOAD_FAST_CHECK_r23 1092 -#define _LOAD_FAST_LOAD_FAST_r02 1093 -#define _LOAD_FAST_LOAD_FAST_r13 1094 -#define _LOAD_FROM_DICT_OR_DEREF_r11 1095 -#define _LOAD_FROM_DICT_OR_GLOBALS_r11 1096 -#define _LOAD_GLOBAL_r00 1097 -#define _LOAD_GLOBAL_BUILTINS_r01 1098 -#define _LOAD_GLOBAL_MODULE_r01 1099 -#define _LOAD_LOCALS_r01 1100 -#define _LOAD_LOCALS_r12 1101 -#define _LOAD_LOCALS_r23 1102 -#define _LOAD_NAME_r01 1103 -#define _LOAD_SMALL_INT_r01 1104 -#define _LOAD_SMALL_INT_r12 1105 -#define _LOAD_SMALL_INT_r23 1106 -#define _LOAD_SMALL_INT_0_r01 1107 -#define _LOAD_SMALL_INT_0_r12 1108 -#define _LOAD_SMALL_INT_0_r23 1109 -#define _LOAD_SMALL_INT_1_r01 1110 -#define _LOAD_SMALL_INT_1_r12 1111 -#define _LOAD_SMALL_INT_1_r23 1112 -#define _LOAD_SMALL_INT_2_r01 1113 -#define _LOAD_SMALL_INT_2_r12 1114 -#define _LOAD_SMALL_INT_2_r23 1115 -#define _LOAD_SMALL_INT_3_r01 1116 -#define _LOAD_SMALL_INT_3_r12 1117 -#define _LOAD_SMALL_INT_3_r23 1118 -#define _LOAD_SPECIAL_r00 1119 -#define _LOAD_SUPER_ATTR_ATTR_r31 1120 -#define _LOAD_SUPER_ATTR_METHOD_r32 1121 -#define _MAKE_CALLARGS_A_TUPLE_r33 1122 -#define _MAKE_CELL_r00 1123 -#define _MAKE_FUNCTION_r11 1124 -#define _MAKE_WARM_r00 1125 -#define _MAKE_WARM_r11 1126 -#define _MAKE_WARM_r22 1127 -#define _MAKE_WARM_r33 1128 -#define _MAP_ADD_r20 1129 -#define _MATCH_CLASS_r31 1130 -#define _MATCH_KEYS_r23 1131 -#define _MATCH_MAPPING_r02 1132 -#define _MATCH_MAPPING_r12 1133 -#define _MATCH_MAPPING_r23 1134 -#define _MATCH_SEQUENCE_r02 1135 -#define _MATCH_SEQUENCE_r12 1136 -#define _MATCH_SEQUENCE_r23 1137 -#define _MAYBE_EXPAND_METHOD_r00 1138 -#define _MAYBE_EXPAND_METHOD_KW_r11 1139 -#define _MONITOR_CALL_r00 1140 -#define _MONITOR_CALL_KW_r11 1141 -#define _MONITOR_JUMP_BACKWARD_r00 1142 -#define _MONITOR_JUMP_BACKWARD_r11 1143 -#define _MONITOR_JUMP_BACKWARD_r22 1144 -#define _MONITOR_JUMP_BACKWARD_r33 1145 -#define _MONITOR_RESUME_r00 1146 -#define _NOP_r00 1147 -#define _NOP_r11 1148 -#define _NOP_r22 1149 -#define _NOP_r33 1150 -#define _POP_CALL_r20 1151 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW_r21 1152 -#define _POP_CALL_ONE_r30 1153 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW_r31 1154 -#define _POP_CALL_TWO_r30 1155 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW_r31 1156 -#define _POP_EXCEPT_r10 1157 -#define _POP_ITER_r20 1158 -#define _POP_JUMP_IF_FALSE_r00 1159 -#define _POP_JUMP_IF_FALSE_r10 1160 -#define _POP_JUMP_IF_FALSE_r21 1161 -#define _POP_JUMP_IF_FALSE_r32 1162 -#define _POP_JUMP_IF_TRUE_r00 1163 -#define _POP_JUMP_IF_TRUE_r10 1164 -#define _POP_JUMP_IF_TRUE_r21 1165 -#define _POP_JUMP_IF_TRUE_r32 1166 -#define _POP_TOP_r10 1167 -#define _POP_TOP_FLOAT_r00 1168 -#define _POP_TOP_FLOAT_r10 1169 -#define _POP_TOP_FLOAT_r21 1170 -#define _POP_TOP_FLOAT_r32 1171 -#define _POP_TOP_INT_r00 1172 -#define _POP_TOP_INT_r10 1173 -#define _POP_TOP_INT_r21 1174 -#define _POP_TOP_INT_r32 1175 -#define _POP_TOP_LOAD_CONST_INLINE_r11 1176 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW_r11 1177 -#define _POP_TOP_NOP_r00 1178 -#define _POP_TOP_NOP_r10 1179 -#define _POP_TOP_NOP_r21 1180 -#define _POP_TOP_NOP_r32 1181 -#define _POP_TOP_UNICODE_r00 1182 -#define _POP_TOP_UNICODE_r10 1183 -#define _POP_TOP_UNICODE_r21 1184 -#define _POP_TOP_UNICODE_r32 1185 -#define _POP_TWO_r20 1186 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW_r21 1187 -#define _PUSH_EXC_INFO_r02 1188 -#define _PUSH_EXC_INFO_r12 1189 -#define _PUSH_EXC_INFO_r23 1190 -#define _PUSH_FRAME_r10 1191 -#define _PUSH_NULL_r01 1192 -#define _PUSH_NULL_r12 1193 -#define _PUSH_NULL_r23 1194 -#define _PUSH_NULL_CONDITIONAL_r00 1195 -#define _PY_FRAME_GENERAL_r01 1196 -#define _PY_FRAME_KW_r11 1197 -#define _QUICKEN_RESUME_r00 1198 -#define _QUICKEN_RESUME_r11 1199 -#define _QUICKEN_RESUME_r22 1200 -#define _QUICKEN_RESUME_r33 1201 -#define _REPLACE_WITH_TRUE_r11 1202 -#define _RESUME_CHECK_r00 1203 -#define _RESUME_CHECK_r11 1204 -#define _RESUME_CHECK_r22 1205 -#define _RESUME_CHECK_r33 1206 -#define _RETURN_GENERATOR_r01 1207 -#define _RETURN_VALUE_r11 1208 -#define _SAVE_RETURN_OFFSET_r00 1209 -#define _SAVE_RETURN_OFFSET_r11 1210 -#define _SAVE_RETURN_OFFSET_r22 1211 -#define _SAVE_RETURN_OFFSET_r33 1212 -#define _SEND_r22 1213 -#define _SEND_GEN_FRAME_r22 1214 -#define _SETUP_ANNOTATIONS_r00 1215 -#define _SET_ADD_r10 1216 -#define _SET_FUNCTION_ATTRIBUTE_r01 1217 -#define _SET_FUNCTION_ATTRIBUTE_r11 1218 -#define _SET_FUNCTION_ATTRIBUTE_r21 1219 -#define _SET_FUNCTION_ATTRIBUTE_r32 1220 -#define _SET_IP_r00 1221 -#define _SET_IP_r11 1222 -#define _SET_IP_r22 1223 -#define _SET_IP_r33 1224 -#define _SET_UPDATE_r10 1225 -#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r02 1226 -#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r12 1227 -#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r22 1228 -#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r32 1229 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03 1230 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13 1231 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23 1232 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33 1233 -#define _SPILL_OR_RELOAD_r01 1234 -#define _SPILL_OR_RELOAD_r02 1235 -#define _SPILL_OR_RELOAD_r03 1236 -#define _SPILL_OR_RELOAD_r10 1237 -#define _SPILL_OR_RELOAD_r12 1238 -#define _SPILL_OR_RELOAD_r13 1239 -#define _SPILL_OR_RELOAD_r20 1240 -#define _SPILL_OR_RELOAD_r21 1241 -#define _SPILL_OR_RELOAD_r23 1242 -#define _SPILL_OR_RELOAD_r30 1243 -#define _SPILL_OR_RELOAD_r31 1244 -#define _SPILL_OR_RELOAD_r32 1245 -#define _START_EXECUTOR_r00 1246 -#define _STORE_ATTR_r20 1247 -#define _STORE_ATTR_INSTANCE_VALUE_r21 1248 -#define _STORE_ATTR_SLOT_r21 1249 -#define _STORE_ATTR_WITH_HINT_r21 1250 -#define _STORE_DEREF_r10 1251 -#define _STORE_FAST_r10 1252 -#define _STORE_FAST_0_r10 1253 -#define _STORE_FAST_1_r10 1254 -#define _STORE_FAST_2_r10 1255 -#define _STORE_FAST_3_r10 1256 -#define _STORE_FAST_4_r10 1257 -#define _STORE_FAST_5_r10 1258 -#define _STORE_FAST_6_r10 1259 -#define _STORE_FAST_7_r10 1260 -#define _STORE_FAST_LOAD_FAST_r11 1261 -#define _STORE_FAST_STORE_FAST_r20 1262 -#define _STORE_GLOBAL_r10 1263 -#define _STORE_NAME_r10 1264 -#define _STORE_SLICE_r30 1265 -#define _STORE_SUBSCR_r30 1266 -#define _STORE_SUBSCR_DICT_r31 1267 -#define _STORE_SUBSCR_LIST_INT_r32 1268 -#define _SWAP_r11 1269 -#define _SWAP_2_r02 1270 -#define _SWAP_2_r12 1271 -#define _SWAP_2_r22 1272 -#define _SWAP_2_r33 1273 -#define _SWAP_3_r03 1274 -#define _SWAP_3_r13 1275 -#define _SWAP_3_r23 1276 -#define _SWAP_3_r33 1277 -#define _TIER2_RESUME_CHECK_r00 1278 -#define _TIER2_RESUME_CHECK_r11 1279 -#define _TIER2_RESUME_CHECK_r22 1280 -#define _TIER2_RESUME_CHECK_r33 1281 -#define _TO_BOOL_r11 1282 -#define _TO_BOOL_BOOL_r01 1283 -#define _TO_BOOL_BOOL_r11 1284 -#define _TO_BOOL_BOOL_r22 1285 -#define _TO_BOOL_BOOL_r33 1286 -#define _TO_BOOL_INT_r11 1287 -#define _TO_BOOL_LIST_r11 1288 -#define _TO_BOOL_NONE_r01 1289 -#define _TO_BOOL_NONE_r11 1290 -#define _TO_BOOL_NONE_r22 1291 -#define _TO_BOOL_NONE_r33 1292 -#define _TO_BOOL_STR_r11 1293 -#define _TRACE_RECORD_r00 1294 -#define _UNARY_INVERT_r11 1295 -#define _UNARY_NEGATIVE_r11 1296 -#define _UNARY_NOT_r01 1297 -#define _UNARY_NOT_r11 1298 -#define _UNARY_NOT_r22 1299 -#define _UNARY_NOT_r33 1300 -#define _UNPACK_EX_r10 1301 -#define _UNPACK_SEQUENCE_r10 1302 -#define _UNPACK_SEQUENCE_LIST_r10 1303 -#define _UNPACK_SEQUENCE_TUPLE_r10 1304 -#define _UNPACK_SEQUENCE_TWO_TUPLE_r12 1305 -#define _WITH_EXCEPT_START_r33 1306 -#define _YIELD_VALUE_r11 1307 -#define MAX_UOP_REGS_ID 1307 +#define MAX_UOP_ID 557 +#define _BINARY_OP_r21 558 +#define _BINARY_OP_ADD_FLOAT_r03 559 +#define _BINARY_OP_ADD_FLOAT_r13 560 +#define _BINARY_OP_ADD_FLOAT_r23 561 +#define _BINARY_OP_ADD_INT_r03 562 +#define _BINARY_OP_ADD_INT_r13 563 +#define _BINARY_OP_ADD_INT_r23 564 +#define _BINARY_OP_ADD_UNICODE_r03 565 +#define _BINARY_OP_ADD_UNICODE_r13 566 +#define _BINARY_OP_ADD_UNICODE_r23 567 +#define _BINARY_OP_EXTEND_r21 568 +#define _BINARY_OP_INPLACE_ADD_UNICODE_r21 569 +#define _BINARY_OP_MULTIPLY_FLOAT_r03 570 +#define _BINARY_OP_MULTIPLY_FLOAT_r13 571 +#define _BINARY_OP_MULTIPLY_FLOAT_r23 572 +#define _BINARY_OP_MULTIPLY_INT_r03 573 +#define _BINARY_OP_MULTIPLY_INT_r13 574 +#define _BINARY_OP_MULTIPLY_INT_r23 575 +#define _BINARY_OP_SUBSCR_CHECK_FUNC_r23 576 +#define _BINARY_OP_SUBSCR_DICT_r21 577 +#define _BINARY_OP_SUBSCR_INIT_CALL_r01 578 +#define _BINARY_OP_SUBSCR_INIT_CALL_r11 579 +#define _BINARY_OP_SUBSCR_INIT_CALL_r21 580 +#define _BINARY_OP_SUBSCR_INIT_CALL_r31 581 +#define _BINARY_OP_SUBSCR_LIST_INT_r23 582 +#define _BINARY_OP_SUBSCR_LIST_SLICE_r21 583 +#define _BINARY_OP_SUBSCR_STR_INT_r23 584 +#define _BINARY_OP_SUBSCR_TUPLE_INT_r03 585 +#define _BINARY_OP_SUBSCR_TUPLE_INT_r13 586 +#define _BINARY_OP_SUBSCR_TUPLE_INT_r23 587 +#define _BINARY_OP_SUBTRACT_FLOAT_r03 588 +#define _BINARY_OP_SUBTRACT_FLOAT_r13 589 +#define _BINARY_OP_SUBTRACT_FLOAT_r23 590 +#define _BINARY_OP_SUBTRACT_INT_r03 591 +#define _BINARY_OP_SUBTRACT_INT_r13 592 +#define _BINARY_OP_SUBTRACT_INT_r23 593 +#define _BINARY_SLICE_r31 594 +#define _BUILD_INTERPOLATION_r01 595 +#define _BUILD_LIST_r01 596 +#define _BUILD_MAP_r01 597 +#define _BUILD_SET_r01 598 +#define _BUILD_SLICE_r01 599 +#define _BUILD_STRING_r01 600 +#define _BUILD_TEMPLATE_r21 601 +#define _BUILD_TUPLE_r01 602 +#define _CALL_BUILTIN_CLASS_r01 603 +#define _CALL_BUILTIN_FAST_r01 604 +#define _CALL_BUILTIN_FAST_WITH_KEYWORDS_r01 605 +#define _CALL_BUILTIN_O_r03 606 +#define _CALL_INTRINSIC_1_r11 607 +#define _CALL_INTRINSIC_2_r21 608 +#define _CALL_ISINSTANCE_r31 609 +#define _CALL_KW_NON_PY_r11 610 +#define _CALL_LEN_r33 611 +#define _CALL_LIST_APPEND_r03 612 +#define _CALL_LIST_APPEND_r13 613 +#define _CALL_LIST_APPEND_r23 614 +#define _CALL_LIST_APPEND_r33 615 +#define _CALL_METHOD_DESCRIPTOR_FAST_r01 616 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r01 617 +#define _CALL_METHOD_DESCRIPTOR_NOARGS_r01 618 +#define _CALL_METHOD_DESCRIPTOR_O_r01 619 +#define _CALL_NON_PY_GENERAL_r01 620 +#define _CALL_STR_1_r32 621 +#define _CALL_TUPLE_1_r32 622 +#define _CALL_TYPE_1_r02 623 +#define _CALL_TYPE_1_r12 624 +#define _CALL_TYPE_1_r22 625 +#define _CALL_TYPE_1_r32 626 +#define _CHECK_AND_ALLOCATE_OBJECT_r00 627 +#define _CHECK_ATTR_CLASS_r01 628 +#define _CHECK_ATTR_CLASS_r11 629 +#define _CHECK_ATTR_CLASS_r22 630 +#define _CHECK_ATTR_CLASS_r33 631 +#define _CHECK_ATTR_METHOD_LAZY_DICT_r01 632 +#define _CHECK_ATTR_METHOD_LAZY_DICT_r11 633 +#define _CHECK_ATTR_METHOD_LAZY_DICT_r22 634 +#define _CHECK_ATTR_METHOD_LAZY_DICT_r33 635 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00 636 +#define _CHECK_EG_MATCH_r22 637 +#define _CHECK_EXC_MATCH_r22 638 +#define _CHECK_FUNCTION_EXACT_ARGS_r00 639 +#define _CHECK_FUNCTION_VERSION_r00 640 +#define _CHECK_FUNCTION_VERSION_INLINE_r00 641 +#define _CHECK_FUNCTION_VERSION_INLINE_r11 642 +#define _CHECK_FUNCTION_VERSION_INLINE_r22 643 +#define _CHECK_FUNCTION_VERSION_INLINE_r33 644 +#define _CHECK_FUNCTION_VERSION_KW_r11 645 +#define _CHECK_IS_NOT_PY_CALLABLE_r00 646 +#define _CHECK_IS_NOT_PY_CALLABLE_KW_r11 647 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r01 648 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r11 649 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r22 650 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r33 651 +#define _CHECK_METHOD_VERSION_r00 652 +#define _CHECK_METHOD_VERSION_KW_r11 653 +#define _CHECK_PEP_523_r00 654 +#define _CHECK_PEP_523_r11 655 +#define _CHECK_PEP_523_r22 656 +#define _CHECK_PEP_523_r33 657 +#define _CHECK_PERIODIC_r00 658 +#define _CHECK_PERIODIC_AT_END_r00 659 +#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00 660 +#define _CHECK_RECURSION_REMAINING_r00 661 +#define _CHECK_RECURSION_REMAINING_r11 662 +#define _CHECK_RECURSION_REMAINING_r22 663 +#define _CHECK_RECURSION_REMAINING_r33 664 +#define _CHECK_STACK_SPACE_r00 665 +#define _CHECK_STACK_SPACE_OPERAND_r00 666 +#define _CHECK_STACK_SPACE_OPERAND_r11 667 +#define _CHECK_STACK_SPACE_OPERAND_r22 668 +#define _CHECK_STACK_SPACE_OPERAND_r33 669 +#define _CHECK_VALIDITY_r00 670 +#define _CHECK_VALIDITY_r11 671 +#define _CHECK_VALIDITY_r22 672 +#define _CHECK_VALIDITY_r33 673 +#define _COLD_DYNAMIC_EXIT_r00 674 +#define _COLD_EXIT_r00 675 +#define _COMPARE_OP_r21 676 +#define _COMPARE_OP_FLOAT_r03 677 +#define _COMPARE_OP_FLOAT_r13 678 +#define _COMPARE_OP_FLOAT_r23 679 +#define _COMPARE_OP_INT_r23 680 +#define _COMPARE_OP_STR_r23 681 +#define _CONTAINS_OP_r21 682 +#define _CONTAINS_OP_DICT_r21 683 +#define _CONTAINS_OP_SET_r21 684 +#define _CONVERT_VALUE_r11 685 +#define _COPY_r01 686 +#define _COPY_1_r02 687 +#define _COPY_1_r12 688 +#define _COPY_1_r23 689 +#define _COPY_2_r03 690 +#define _COPY_2_r13 691 +#define _COPY_2_r23 692 +#define _COPY_3_r03 693 +#define _COPY_3_r13 694 +#define _COPY_3_r23 695 +#define _COPY_3_r33 696 +#define _COPY_FREE_VARS_r00 697 +#define _COPY_FREE_VARS_r11 698 +#define _COPY_FREE_VARS_r22 699 +#define _COPY_FREE_VARS_r33 700 +#define _CREATE_INIT_FRAME_r01 701 +#define _DELETE_ATTR_r10 702 +#define _DELETE_DEREF_r00 703 +#define _DELETE_FAST_r00 704 +#define _DELETE_GLOBAL_r00 705 +#define _DELETE_NAME_r00 706 +#define _DELETE_SUBSCR_r20 707 +#define _DEOPT_r00 708 +#define _DEOPT_r10 709 +#define _DEOPT_r20 710 +#define _DEOPT_r30 711 +#define _DICT_MERGE_r10 712 +#define _DICT_UPDATE_r10 713 +#define _DO_CALL_r01 714 +#define _DO_CALL_FUNCTION_EX_r31 715 +#define _DO_CALL_KW_r11 716 +#define _DYNAMIC_EXIT_r00 717 +#define _DYNAMIC_EXIT_r10 718 +#define _DYNAMIC_EXIT_r20 719 +#define _DYNAMIC_EXIT_r30 720 +#define _END_FOR_r10 721 +#define _END_SEND_r21 722 +#define _ERROR_POP_N_r00 723 +#define _EXIT_INIT_CHECK_r10 724 +#define _EXIT_TRACE_r00 725 +#define _EXIT_TRACE_r10 726 +#define _EXIT_TRACE_r20 727 +#define _EXIT_TRACE_r30 728 +#define _EXPAND_METHOD_r00 729 +#define _EXPAND_METHOD_KW_r11 730 +#define _FATAL_ERROR_r00 731 +#define _FATAL_ERROR_r11 732 +#define _FATAL_ERROR_r22 733 +#define _FATAL_ERROR_r33 734 +#define _FORMAT_SIMPLE_r11 735 +#define _FORMAT_WITH_SPEC_r21 736 +#define _FOR_ITER_r23 737 +#define _FOR_ITER_GEN_FRAME_r03 738 +#define _FOR_ITER_GEN_FRAME_r13 739 +#define _FOR_ITER_GEN_FRAME_r23 740 +#define _FOR_ITER_TIER_TWO_r23 741 +#define _GET_AITER_r11 742 +#define _GET_ANEXT_r12 743 +#define _GET_AWAITABLE_r11 744 +#define _GET_ITER_r12 745 +#define _GET_LEN_r12 746 +#define _GET_YIELD_FROM_ITER_r11 747 +#define _GUARD_BINARY_OP_EXTEND_r22 748 +#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r02 749 +#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r12 750 +#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r22 751 +#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r33 752 +#define _GUARD_CALLABLE_ISINSTANCE_r03 753 +#define _GUARD_CALLABLE_ISINSTANCE_r13 754 +#define _GUARD_CALLABLE_ISINSTANCE_r23 755 +#define _GUARD_CALLABLE_ISINSTANCE_r33 756 +#define _GUARD_CALLABLE_LEN_r03 757 +#define _GUARD_CALLABLE_LEN_r13 758 +#define _GUARD_CALLABLE_LEN_r23 759 +#define _GUARD_CALLABLE_LEN_r33 760 +#define _GUARD_CALLABLE_LIST_APPEND_r03 761 +#define _GUARD_CALLABLE_LIST_APPEND_r13 762 +#define _GUARD_CALLABLE_LIST_APPEND_r23 763 +#define _GUARD_CALLABLE_LIST_APPEND_r33 764 +#define _GUARD_CALLABLE_STR_1_r03 765 +#define _GUARD_CALLABLE_STR_1_r13 766 +#define _GUARD_CALLABLE_STR_1_r23 767 +#define _GUARD_CALLABLE_STR_1_r33 768 +#define _GUARD_CALLABLE_TUPLE_1_r03 769 +#define _GUARD_CALLABLE_TUPLE_1_r13 770 +#define _GUARD_CALLABLE_TUPLE_1_r23 771 +#define _GUARD_CALLABLE_TUPLE_1_r33 772 +#define _GUARD_CALLABLE_TYPE_1_r03 773 +#define _GUARD_CALLABLE_TYPE_1_r13 774 +#define _GUARD_CALLABLE_TYPE_1_r23 775 +#define _GUARD_CALLABLE_TYPE_1_r33 776 +#define _GUARD_DORV_NO_DICT_r01 777 +#define _GUARD_DORV_NO_DICT_r11 778 +#define _GUARD_DORV_NO_DICT_r22 779 +#define _GUARD_DORV_NO_DICT_r33 780 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r01 781 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11 782 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22 783 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33 784 +#define _GUARD_GLOBALS_VERSION_r00 785 +#define _GUARD_GLOBALS_VERSION_r11 786 +#define _GUARD_GLOBALS_VERSION_r22 787 +#define _GUARD_GLOBALS_VERSION_r33 788 +#define _GUARD_IP_RETURN_GENERATOR_r00 789 +#define _GUARD_IP_RETURN_GENERATOR_r11 790 +#define _GUARD_IP_RETURN_GENERATOR_r22 791 +#define _GUARD_IP_RETURN_GENERATOR_r33 792 +#define _GUARD_IP_RETURN_VALUE_r00 793 +#define _GUARD_IP_RETURN_VALUE_r11 794 +#define _GUARD_IP_RETURN_VALUE_r22 795 +#define _GUARD_IP_RETURN_VALUE_r33 796 +#define _GUARD_IP_YIELD_VALUE_r00 797 +#define _GUARD_IP_YIELD_VALUE_r11 798 +#define _GUARD_IP_YIELD_VALUE_r22 799 +#define _GUARD_IP_YIELD_VALUE_r33 800 +#define _GUARD_IP__PUSH_FRAME_r00 801 +#define _GUARD_IP__PUSH_FRAME_r11 802 +#define _GUARD_IP__PUSH_FRAME_r22 803 +#define _GUARD_IP__PUSH_FRAME_r33 804 +#define _GUARD_IS_FALSE_POP_r00 805 +#define _GUARD_IS_FALSE_POP_r10 806 +#define _GUARD_IS_FALSE_POP_r21 807 +#define _GUARD_IS_FALSE_POP_r32 808 +#define _GUARD_IS_NONE_POP_r00 809 +#define _GUARD_IS_NONE_POP_r10 810 +#define _GUARD_IS_NONE_POP_r21 811 +#define _GUARD_IS_NONE_POP_r32 812 +#define _GUARD_IS_NOT_NONE_POP_r10 813 +#define _GUARD_IS_TRUE_POP_r00 814 +#define _GUARD_IS_TRUE_POP_r10 815 +#define _GUARD_IS_TRUE_POP_r21 816 +#define _GUARD_IS_TRUE_POP_r32 817 +#define _GUARD_KEYS_VERSION_r01 818 +#define _GUARD_KEYS_VERSION_r11 819 +#define _GUARD_KEYS_VERSION_r22 820 +#define _GUARD_KEYS_VERSION_r33 821 +#define _GUARD_NOS_DICT_r02 822 +#define _GUARD_NOS_DICT_r12 823 +#define _GUARD_NOS_DICT_r22 824 +#define _GUARD_NOS_DICT_r33 825 +#define _GUARD_NOS_FLOAT_r02 826 +#define _GUARD_NOS_FLOAT_r12 827 +#define _GUARD_NOS_FLOAT_r22 828 +#define _GUARD_NOS_FLOAT_r33 829 +#define _GUARD_NOS_INT_r02 830 +#define _GUARD_NOS_INT_r12 831 +#define _GUARD_NOS_INT_r22 832 +#define _GUARD_NOS_INT_r33 833 +#define _GUARD_NOS_LIST_r02 834 +#define _GUARD_NOS_LIST_r12 835 +#define _GUARD_NOS_LIST_r22 836 +#define _GUARD_NOS_LIST_r33 837 +#define _GUARD_NOS_NOT_NULL_r02 838 +#define _GUARD_NOS_NOT_NULL_r12 839 +#define _GUARD_NOS_NOT_NULL_r22 840 +#define _GUARD_NOS_NOT_NULL_r33 841 +#define _GUARD_NOS_NULL_r02 842 +#define _GUARD_NOS_NULL_r12 843 +#define _GUARD_NOS_NULL_r22 844 +#define _GUARD_NOS_NULL_r33 845 +#define _GUARD_NOS_OVERFLOWED_r02 846 +#define _GUARD_NOS_OVERFLOWED_r12 847 +#define _GUARD_NOS_OVERFLOWED_r22 848 +#define _GUARD_NOS_OVERFLOWED_r33 849 +#define _GUARD_NOS_TUPLE_r02 850 +#define _GUARD_NOS_TUPLE_r12 851 +#define _GUARD_NOS_TUPLE_r22 852 +#define _GUARD_NOS_TUPLE_r33 853 +#define _GUARD_NOS_UNICODE_r02 854 +#define _GUARD_NOS_UNICODE_r12 855 +#define _GUARD_NOS_UNICODE_r22 856 +#define _GUARD_NOS_UNICODE_r33 857 +#define _GUARD_NOT_EXHAUSTED_LIST_r02 858 +#define _GUARD_NOT_EXHAUSTED_LIST_r12 859 +#define _GUARD_NOT_EXHAUSTED_LIST_r22 860 +#define _GUARD_NOT_EXHAUSTED_LIST_r33 861 +#define _GUARD_NOT_EXHAUSTED_RANGE_r02 862 +#define _GUARD_NOT_EXHAUSTED_RANGE_r12 863 +#define _GUARD_NOT_EXHAUSTED_RANGE_r22 864 +#define _GUARD_NOT_EXHAUSTED_RANGE_r33 865 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r02 866 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r12 867 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r22 868 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r33 869 +#define _GUARD_THIRD_NULL_r03 870 +#define _GUARD_THIRD_NULL_r13 871 +#define _GUARD_THIRD_NULL_r23 872 +#define _GUARD_THIRD_NULL_r33 873 +#define _GUARD_TOS_ANY_SET_r01 874 +#define _GUARD_TOS_ANY_SET_r11 875 +#define _GUARD_TOS_ANY_SET_r22 876 +#define _GUARD_TOS_ANY_SET_r33 877 +#define _GUARD_TOS_DICT_r01 878 +#define _GUARD_TOS_DICT_r11 879 +#define _GUARD_TOS_DICT_r22 880 +#define _GUARD_TOS_DICT_r33 881 +#define _GUARD_TOS_FLOAT_r01 882 +#define _GUARD_TOS_FLOAT_r11 883 +#define _GUARD_TOS_FLOAT_r22 884 +#define _GUARD_TOS_FLOAT_r33 885 +#define _GUARD_TOS_INT_r01 886 +#define _GUARD_TOS_INT_r11 887 +#define _GUARD_TOS_INT_r22 888 +#define _GUARD_TOS_INT_r33 889 +#define _GUARD_TOS_LIST_r01 890 +#define _GUARD_TOS_LIST_r11 891 +#define _GUARD_TOS_LIST_r22 892 +#define _GUARD_TOS_LIST_r33 893 +#define _GUARD_TOS_OVERFLOWED_r01 894 +#define _GUARD_TOS_OVERFLOWED_r11 895 +#define _GUARD_TOS_OVERFLOWED_r22 896 +#define _GUARD_TOS_OVERFLOWED_r33 897 +#define _GUARD_TOS_SLICE_r01 898 +#define _GUARD_TOS_SLICE_r11 899 +#define _GUARD_TOS_SLICE_r22 900 +#define _GUARD_TOS_SLICE_r33 901 +#define _GUARD_TOS_TUPLE_r01 902 +#define _GUARD_TOS_TUPLE_r11 903 +#define _GUARD_TOS_TUPLE_r22 904 +#define _GUARD_TOS_TUPLE_r33 905 +#define _GUARD_TOS_UNICODE_r01 906 +#define _GUARD_TOS_UNICODE_r11 907 +#define _GUARD_TOS_UNICODE_r22 908 +#define _GUARD_TOS_UNICODE_r33 909 +#define _GUARD_TYPE_VERSION_r01 910 +#define _GUARD_TYPE_VERSION_r11 911 +#define _GUARD_TYPE_VERSION_r22 912 +#define _GUARD_TYPE_VERSION_r33 913 +#define _GUARD_TYPE_VERSION_AND_LOCK_r01 914 +#define _GUARD_TYPE_VERSION_AND_LOCK_r11 915 +#define _GUARD_TYPE_VERSION_AND_LOCK_r22 916 +#define _GUARD_TYPE_VERSION_AND_LOCK_r33 917 +#define _HANDLE_PENDING_AND_DEOPT_r00 918 +#define _HANDLE_PENDING_AND_DEOPT_r10 919 +#define _HANDLE_PENDING_AND_DEOPT_r20 920 +#define _HANDLE_PENDING_AND_DEOPT_r30 921 +#define _IMPORT_FROM_r12 922 +#define _IMPORT_NAME_r21 923 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00 924 +#define _INIT_CALL_PY_EXACT_ARGS_r01 925 +#define _INIT_CALL_PY_EXACT_ARGS_0_r01 926 +#define _INIT_CALL_PY_EXACT_ARGS_1_r01 927 +#define _INIT_CALL_PY_EXACT_ARGS_2_r01 928 +#define _INIT_CALL_PY_EXACT_ARGS_3_r01 929 +#define _INIT_CALL_PY_EXACT_ARGS_4_r01 930 +#define _INSERT_NULL_r10 931 +#define _INSTRUMENTED_FOR_ITER_r23 932 +#define _INSTRUMENTED_INSTRUCTION_r00 933 +#define _INSTRUMENTED_JUMP_FORWARD_r00 934 +#define _INSTRUMENTED_JUMP_FORWARD_r11 935 +#define _INSTRUMENTED_JUMP_FORWARD_r22 936 +#define _INSTRUMENTED_JUMP_FORWARD_r33 937 +#define _INSTRUMENTED_LINE_r00 938 +#define _INSTRUMENTED_NOT_TAKEN_r00 939 +#define _INSTRUMENTED_NOT_TAKEN_r11 940 +#define _INSTRUMENTED_NOT_TAKEN_r22 941 +#define _INSTRUMENTED_NOT_TAKEN_r33 942 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r00 943 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r10 944 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r21 945 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r32 946 +#define _INSTRUMENTED_POP_JUMP_IF_NONE_r10 947 +#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE_r10 948 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r00 949 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r10 950 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r21 951 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r32 952 +#define _IS_NONE_r11 953 +#define _IS_OP_r03 954 +#define _IS_OP_r13 955 +#define _IS_OP_r23 956 +#define _ITER_CHECK_LIST_r02 957 +#define _ITER_CHECK_LIST_r12 958 +#define _ITER_CHECK_LIST_r22 959 +#define _ITER_CHECK_LIST_r33 960 +#define _ITER_CHECK_RANGE_r02 961 +#define _ITER_CHECK_RANGE_r12 962 +#define _ITER_CHECK_RANGE_r22 963 +#define _ITER_CHECK_RANGE_r33 964 +#define _ITER_CHECK_TUPLE_r02 965 +#define _ITER_CHECK_TUPLE_r12 966 +#define _ITER_CHECK_TUPLE_r22 967 +#define _ITER_CHECK_TUPLE_r33 968 +#define _ITER_JUMP_LIST_r02 969 +#define _ITER_JUMP_LIST_r12 970 +#define _ITER_JUMP_LIST_r22 971 +#define _ITER_JUMP_LIST_r33 972 +#define _ITER_JUMP_RANGE_r02 973 +#define _ITER_JUMP_RANGE_r12 974 +#define _ITER_JUMP_RANGE_r22 975 +#define _ITER_JUMP_RANGE_r33 976 +#define _ITER_JUMP_TUPLE_r02 977 +#define _ITER_JUMP_TUPLE_r12 978 +#define _ITER_JUMP_TUPLE_r22 979 +#define _ITER_JUMP_TUPLE_r33 980 +#define _ITER_NEXT_LIST_r23 981 +#define _ITER_NEXT_LIST_TIER_TWO_r23 982 +#define _ITER_NEXT_RANGE_r03 983 +#define _ITER_NEXT_RANGE_r13 984 +#define _ITER_NEXT_RANGE_r23 985 +#define _ITER_NEXT_TUPLE_r03 986 +#define _ITER_NEXT_TUPLE_r13 987 +#define _ITER_NEXT_TUPLE_r23 988 +#define _JUMP_BACKWARD_NO_INTERRUPT_r00 989 +#define _JUMP_BACKWARD_NO_INTERRUPT_r11 990 +#define _JUMP_BACKWARD_NO_INTERRUPT_r22 991 +#define _JUMP_BACKWARD_NO_INTERRUPT_r33 992 +#define _JUMP_TO_TOP_r00 993 +#define _LIST_APPEND_r10 994 +#define _LIST_EXTEND_r10 995 +#define _LOAD_ATTR_r10 996 +#define _LOAD_ATTR_CLASS_r11 997 +#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_r11 998 +#define _LOAD_ATTR_INSTANCE_VALUE_r02 999 +#define _LOAD_ATTR_INSTANCE_VALUE_r12 1000 +#define _LOAD_ATTR_INSTANCE_VALUE_r23 1001 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r02 1002 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r12 1003 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r23 1004 +#define _LOAD_ATTR_METHOD_NO_DICT_r02 1005 +#define _LOAD_ATTR_METHOD_NO_DICT_r12 1006 +#define _LOAD_ATTR_METHOD_NO_DICT_r23 1007 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r02 1008 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r12 1009 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r23 1010 +#define _LOAD_ATTR_MODULE_r11 1011 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11 1012 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11 1013 +#define _LOAD_ATTR_PROPERTY_FRAME_r11 1014 +#define _LOAD_ATTR_SLOT_r11 1015 +#define _LOAD_ATTR_WITH_HINT_r12 1016 +#define _LOAD_BUILD_CLASS_r01 1017 +#define _LOAD_BYTECODE_r00 1018 +#define _LOAD_COMMON_CONSTANT_r01 1019 +#define _LOAD_COMMON_CONSTANT_r12 1020 +#define _LOAD_COMMON_CONSTANT_r23 1021 +#define _LOAD_CONST_r01 1022 +#define _LOAD_CONST_r12 1023 +#define _LOAD_CONST_r23 1024 +#define _LOAD_CONST_INLINE_r01 1025 +#define _LOAD_CONST_INLINE_r12 1026 +#define _LOAD_CONST_INLINE_r23 1027 +#define _LOAD_CONST_INLINE_BORROW_r01 1028 +#define _LOAD_CONST_INLINE_BORROW_r12 1029 +#define _LOAD_CONST_INLINE_BORROW_r23 1030 +#define _LOAD_CONST_UNDER_INLINE_r02 1031 +#define _LOAD_CONST_UNDER_INLINE_r12 1032 +#define _LOAD_CONST_UNDER_INLINE_r23 1033 +#define _LOAD_CONST_UNDER_INLINE_BORROW_r02 1034 +#define _LOAD_CONST_UNDER_INLINE_BORROW_r12 1035 +#define _LOAD_CONST_UNDER_INLINE_BORROW_r23 1036 +#define _LOAD_DEREF_r01 1037 +#define _LOAD_FAST_r01 1038 +#define _LOAD_FAST_r12 1039 +#define _LOAD_FAST_r23 1040 +#define _LOAD_FAST_0_r01 1041 +#define _LOAD_FAST_0_r12 1042 +#define _LOAD_FAST_0_r23 1043 +#define _LOAD_FAST_1_r01 1044 +#define _LOAD_FAST_1_r12 1045 +#define _LOAD_FAST_1_r23 1046 +#define _LOAD_FAST_2_r01 1047 +#define _LOAD_FAST_2_r12 1048 +#define _LOAD_FAST_2_r23 1049 +#define _LOAD_FAST_3_r01 1050 +#define _LOAD_FAST_3_r12 1051 +#define _LOAD_FAST_3_r23 1052 +#define _LOAD_FAST_4_r01 1053 +#define _LOAD_FAST_4_r12 1054 +#define _LOAD_FAST_4_r23 1055 +#define _LOAD_FAST_5_r01 1056 +#define _LOAD_FAST_5_r12 1057 +#define _LOAD_FAST_5_r23 1058 +#define _LOAD_FAST_6_r01 1059 +#define _LOAD_FAST_6_r12 1060 +#define _LOAD_FAST_6_r23 1061 +#define _LOAD_FAST_7_r01 1062 +#define _LOAD_FAST_7_r12 1063 +#define _LOAD_FAST_7_r23 1064 +#define _LOAD_FAST_AND_CLEAR_r01 1065 +#define _LOAD_FAST_AND_CLEAR_r12 1066 +#define _LOAD_FAST_AND_CLEAR_r23 1067 +#define _LOAD_FAST_BORROW_r01 1068 +#define _LOAD_FAST_BORROW_r12 1069 +#define _LOAD_FAST_BORROW_r23 1070 +#define _LOAD_FAST_BORROW_0_r01 1071 +#define _LOAD_FAST_BORROW_0_r12 1072 +#define _LOAD_FAST_BORROW_0_r23 1073 +#define _LOAD_FAST_BORROW_1_r01 1074 +#define _LOAD_FAST_BORROW_1_r12 1075 +#define _LOAD_FAST_BORROW_1_r23 1076 +#define _LOAD_FAST_BORROW_2_r01 1077 +#define _LOAD_FAST_BORROW_2_r12 1078 +#define _LOAD_FAST_BORROW_2_r23 1079 +#define _LOAD_FAST_BORROW_3_r01 1080 +#define _LOAD_FAST_BORROW_3_r12 1081 +#define _LOAD_FAST_BORROW_3_r23 1082 +#define _LOAD_FAST_BORROW_4_r01 1083 +#define _LOAD_FAST_BORROW_4_r12 1084 +#define _LOAD_FAST_BORROW_4_r23 1085 +#define _LOAD_FAST_BORROW_5_r01 1086 +#define _LOAD_FAST_BORROW_5_r12 1087 +#define _LOAD_FAST_BORROW_5_r23 1088 +#define _LOAD_FAST_BORROW_6_r01 1089 +#define _LOAD_FAST_BORROW_6_r12 1090 +#define _LOAD_FAST_BORROW_6_r23 1091 +#define _LOAD_FAST_BORROW_7_r01 1092 +#define _LOAD_FAST_BORROW_7_r12 1093 +#define _LOAD_FAST_BORROW_7_r23 1094 +#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r02 1095 +#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r13 1096 +#define _LOAD_FAST_CHECK_r01 1097 +#define _LOAD_FAST_CHECK_r12 1098 +#define _LOAD_FAST_CHECK_r23 1099 +#define _LOAD_FAST_LOAD_FAST_r02 1100 +#define _LOAD_FAST_LOAD_FAST_r13 1101 +#define _LOAD_FROM_DICT_OR_DEREF_r11 1102 +#define _LOAD_FROM_DICT_OR_GLOBALS_r11 1103 +#define _LOAD_GLOBAL_r00 1104 +#define _LOAD_GLOBAL_BUILTINS_r01 1105 +#define _LOAD_GLOBAL_MODULE_r01 1106 +#define _LOAD_LOCALS_r01 1107 +#define _LOAD_LOCALS_r12 1108 +#define _LOAD_LOCALS_r23 1109 +#define _LOAD_NAME_r01 1110 +#define _LOAD_SMALL_INT_r01 1111 +#define _LOAD_SMALL_INT_r12 1112 +#define _LOAD_SMALL_INT_r23 1113 +#define _LOAD_SMALL_INT_0_r01 1114 +#define _LOAD_SMALL_INT_0_r12 1115 +#define _LOAD_SMALL_INT_0_r23 1116 +#define _LOAD_SMALL_INT_1_r01 1117 +#define _LOAD_SMALL_INT_1_r12 1118 +#define _LOAD_SMALL_INT_1_r23 1119 +#define _LOAD_SMALL_INT_2_r01 1120 +#define _LOAD_SMALL_INT_2_r12 1121 +#define _LOAD_SMALL_INT_2_r23 1122 +#define _LOAD_SMALL_INT_3_r01 1123 +#define _LOAD_SMALL_INT_3_r12 1124 +#define _LOAD_SMALL_INT_3_r23 1125 +#define _LOAD_SPECIAL_r00 1126 +#define _LOAD_SUPER_ATTR_ATTR_r31 1127 +#define _LOAD_SUPER_ATTR_METHOD_r32 1128 +#define _MAKE_CALLARGS_A_TUPLE_r33 1129 +#define _MAKE_CELL_r00 1130 +#define _MAKE_FUNCTION_r11 1131 +#define _MAKE_WARM_r00 1132 +#define _MAKE_WARM_r11 1133 +#define _MAKE_WARM_r22 1134 +#define _MAKE_WARM_r33 1135 +#define _MAP_ADD_r20 1136 +#define _MATCH_CLASS_r31 1137 +#define _MATCH_KEYS_r23 1138 +#define _MATCH_MAPPING_r02 1139 +#define _MATCH_MAPPING_r12 1140 +#define _MATCH_MAPPING_r23 1141 +#define _MATCH_SEQUENCE_r02 1142 +#define _MATCH_SEQUENCE_r12 1143 +#define _MATCH_SEQUENCE_r23 1144 +#define _MAYBE_EXPAND_METHOD_r00 1145 +#define _MAYBE_EXPAND_METHOD_KW_r11 1146 +#define _MONITOR_CALL_r00 1147 +#define _MONITOR_CALL_KW_r11 1148 +#define _MONITOR_JUMP_BACKWARD_r00 1149 +#define _MONITOR_JUMP_BACKWARD_r11 1150 +#define _MONITOR_JUMP_BACKWARD_r22 1151 +#define _MONITOR_JUMP_BACKWARD_r33 1152 +#define _MONITOR_RESUME_r00 1153 +#define _NOP_r00 1154 +#define _NOP_r11 1155 +#define _NOP_r22 1156 +#define _NOP_r33 1157 +#define _POP_CALL_r20 1158 +#define _POP_CALL_LOAD_CONST_INLINE_BORROW_r21 1159 +#define _POP_CALL_ONE_r30 1160 +#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW_r31 1161 +#define _POP_CALL_TWO_r30 1162 +#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW_r31 1163 +#define _POP_EXCEPT_r10 1164 +#define _POP_ITER_r20 1165 +#define _POP_JUMP_IF_FALSE_r00 1166 +#define _POP_JUMP_IF_FALSE_r10 1167 +#define _POP_JUMP_IF_FALSE_r21 1168 +#define _POP_JUMP_IF_FALSE_r32 1169 +#define _POP_JUMP_IF_TRUE_r00 1170 +#define _POP_JUMP_IF_TRUE_r10 1171 +#define _POP_JUMP_IF_TRUE_r21 1172 +#define _POP_JUMP_IF_TRUE_r32 1173 +#define _POP_TOP_r10 1174 +#define _POP_TOP_FLOAT_r00 1175 +#define _POP_TOP_FLOAT_r10 1176 +#define _POP_TOP_FLOAT_r21 1177 +#define _POP_TOP_FLOAT_r32 1178 +#define _POP_TOP_INT_r00 1179 +#define _POP_TOP_INT_r10 1180 +#define _POP_TOP_INT_r21 1181 +#define _POP_TOP_INT_r32 1182 +#define _POP_TOP_LOAD_CONST_INLINE_r11 1183 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW_r11 1184 +#define _POP_TOP_NOP_r00 1185 +#define _POP_TOP_NOP_r10 1186 +#define _POP_TOP_NOP_r21 1187 +#define _POP_TOP_NOP_r32 1188 +#define _POP_TOP_UNICODE_r00 1189 +#define _POP_TOP_UNICODE_r10 1190 +#define _POP_TOP_UNICODE_r21 1191 +#define _POP_TOP_UNICODE_r32 1192 +#define _POP_TWO_r20 1193 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW_r21 1194 +#define _PUSH_EXC_INFO_r02 1195 +#define _PUSH_EXC_INFO_r12 1196 +#define _PUSH_EXC_INFO_r23 1197 +#define _PUSH_FRAME_r10 1198 +#define _PUSH_NULL_r01 1199 +#define _PUSH_NULL_r12 1200 +#define _PUSH_NULL_r23 1201 +#define _PUSH_NULL_CONDITIONAL_r00 1202 +#define _PY_FRAME_GENERAL_r01 1203 +#define _PY_FRAME_KW_r11 1204 +#define _QUICKEN_RESUME_r00 1205 +#define _QUICKEN_RESUME_r11 1206 +#define _QUICKEN_RESUME_r22 1207 +#define _QUICKEN_RESUME_r33 1208 +#define _REPLACE_WITH_TRUE_r11 1209 +#define _RESUME_CHECK_r00 1210 +#define _RESUME_CHECK_r11 1211 +#define _RESUME_CHECK_r22 1212 +#define _RESUME_CHECK_r33 1213 +#define _RETURN_GENERATOR_r01 1214 +#define _RETURN_VALUE_r11 1215 +#define _SAVE_RETURN_OFFSET_r00 1216 +#define _SAVE_RETURN_OFFSET_r11 1217 +#define _SAVE_RETURN_OFFSET_r22 1218 +#define _SAVE_RETURN_OFFSET_r33 1219 +#define _SEND_r22 1220 +#define _SEND_GEN_FRAME_r22 1221 +#define _SETUP_ANNOTATIONS_r00 1222 +#define _SET_ADD_r10 1223 +#define _SET_FUNCTION_ATTRIBUTE_r01 1224 +#define _SET_FUNCTION_ATTRIBUTE_r11 1225 +#define _SET_FUNCTION_ATTRIBUTE_r21 1226 +#define _SET_FUNCTION_ATTRIBUTE_r32 1227 +#define _SET_IP_r00 1228 +#define _SET_IP_r11 1229 +#define _SET_IP_r22 1230 +#define _SET_IP_r33 1231 +#define _SET_UPDATE_r10 1232 +#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r02 1233 +#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r12 1234 +#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r22 1235 +#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r32 1236 +#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03 1237 +#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13 1238 +#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23 1239 +#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33 1240 +#define _SPILL_OR_RELOAD_r01 1241 +#define _SPILL_OR_RELOAD_r02 1242 +#define _SPILL_OR_RELOAD_r03 1243 +#define _SPILL_OR_RELOAD_r10 1244 +#define _SPILL_OR_RELOAD_r12 1245 +#define _SPILL_OR_RELOAD_r13 1246 +#define _SPILL_OR_RELOAD_r20 1247 +#define _SPILL_OR_RELOAD_r21 1248 +#define _SPILL_OR_RELOAD_r23 1249 +#define _SPILL_OR_RELOAD_r30 1250 +#define _SPILL_OR_RELOAD_r31 1251 +#define _SPILL_OR_RELOAD_r32 1252 +#define _START_EXECUTOR_r00 1253 +#define _STORE_ATTR_r20 1254 +#define _STORE_ATTR_INSTANCE_VALUE_r21 1255 +#define _STORE_ATTR_SLOT_r21 1256 +#define _STORE_ATTR_WITH_HINT_r21 1257 +#define _STORE_DEREF_r10 1258 +#define _STORE_FAST_r10 1259 +#define _STORE_FAST_0_r10 1260 +#define _STORE_FAST_1_r10 1261 +#define _STORE_FAST_2_r10 1262 +#define _STORE_FAST_3_r10 1263 +#define _STORE_FAST_4_r10 1264 +#define _STORE_FAST_5_r10 1265 +#define _STORE_FAST_6_r10 1266 +#define _STORE_FAST_7_r10 1267 +#define _STORE_FAST_LOAD_FAST_r11 1268 +#define _STORE_FAST_STORE_FAST_r20 1269 +#define _STORE_GLOBAL_r10 1270 +#define _STORE_NAME_r10 1271 +#define _STORE_SLICE_r30 1272 +#define _STORE_SUBSCR_r30 1273 +#define _STORE_SUBSCR_DICT_r31 1274 +#define _STORE_SUBSCR_LIST_INT_r32 1275 +#define _SWAP_r11 1276 +#define _SWAP_2_r02 1277 +#define _SWAP_2_r12 1278 +#define _SWAP_2_r22 1279 +#define _SWAP_2_r33 1280 +#define _SWAP_3_r03 1281 +#define _SWAP_3_r13 1282 +#define _SWAP_3_r23 1283 +#define _SWAP_3_r33 1284 +#define _TIER2_RESUME_CHECK_r00 1285 +#define _TIER2_RESUME_CHECK_r11 1286 +#define _TIER2_RESUME_CHECK_r22 1287 +#define _TIER2_RESUME_CHECK_r33 1288 +#define _TO_BOOL_r11 1289 +#define _TO_BOOL_BOOL_r01 1290 +#define _TO_BOOL_BOOL_r11 1291 +#define _TO_BOOL_BOOL_r22 1292 +#define _TO_BOOL_BOOL_r33 1293 +#define _TO_BOOL_INT_r11 1294 +#define _TO_BOOL_LIST_r11 1295 +#define _TO_BOOL_NONE_r01 1296 +#define _TO_BOOL_NONE_r11 1297 +#define _TO_BOOL_NONE_r22 1298 +#define _TO_BOOL_NONE_r33 1299 +#define _TO_BOOL_STR_r11 1300 +#define _TRACE_RECORD_r00 1301 +#define _UNARY_INVERT_r11 1302 +#define _UNARY_NEGATIVE_r11 1303 +#define _UNARY_NOT_r01 1304 +#define _UNARY_NOT_r11 1305 +#define _UNARY_NOT_r22 1306 +#define _UNARY_NOT_r33 1307 +#define _UNPACK_EX_r10 1308 +#define _UNPACK_SEQUENCE_r10 1309 +#define _UNPACK_SEQUENCE_LIST_r10 1310 +#define _UNPACK_SEQUENCE_TUPLE_r10 1311 +#define _UNPACK_SEQUENCE_TWO_TUPLE_r12 1312 +#define _WITH_EXCEPT_START_r33 1313 +#define _YIELD_VALUE_r11 1314 +#define MAX_UOP_REGS_ID 1314 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 6262a14e266c4d..1838dd3f0977b2 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -122,7 +122,8 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BINARY_OP_SUBSCR_STR_INT] = HAS_DEOPT_FLAG, [_GUARD_NOS_TUPLE] = HAS_EXIT_FLAG, [_GUARD_TOS_TUPLE] = HAS_EXIT_FLAG, - [_BINARY_OP_SUBSCR_TUPLE_INT] = HAS_DEOPT_FLAG, + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS] = HAS_DEOPT_FLAG, + [_BINARY_OP_SUBSCR_TUPLE_INT] = 0, [_GUARD_NOS_DICT] = HAS_EXIT_FLAG, [_GUARD_TOS_DICT] = HAS_EXIT_FLAG, [_BINARY_OP_SUBSCR_DICT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -1150,11 +1151,20 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { 3, 3, _GUARD_TOS_TUPLE_r33 }, }, }, + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r02 }, + { 2, 1, _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r12 }, + { 2, 2, _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r22 }, + { 3, 3, _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r33 }, + }, + }, [_BINARY_OP_SUBSCR_TUPLE_INT] = { - .best = { 2, 2, 2, 2 }, + .best = { 0, 1, 2, 2 }, .entries = { - { -1, -1, -1 }, - { -1, -1, -1 }, + { 3, 0, _BINARY_OP_SUBSCR_TUPLE_INT_r03 }, + { 3, 1, _BINARY_OP_SUBSCR_TUPLE_INT_r13 }, { 3, 2, _BINARY_OP_SUBSCR_TUPLE_INT_r23 }, { -1, -1, -1 }, }, @@ -3453,6 +3463,12 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_GUARD_TOS_TUPLE_r11] = _GUARD_TOS_TUPLE, [_GUARD_TOS_TUPLE_r22] = _GUARD_TOS_TUPLE, [_GUARD_TOS_TUPLE_r33] = _GUARD_TOS_TUPLE, + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r02] = _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS, + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r12] = _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS, + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r22] = _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS, + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r33] = _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS, + [_BINARY_OP_SUBSCR_TUPLE_INT_r03] = _BINARY_OP_SUBSCR_TUPLE_INT, + [_BINARY_OP_SUBSCR_TUPLE_INT_r13] = _BINARY_OP_SUBSCR_TUPLE_INT, [_BINARY_OP_SUBSCR_TUPLE_INT_r23] = _BINARY_OP_SUBSCR_TUPLE_INT, [_GUARD_NOS_DICT_r02] = _GUARD_NOS_DICT, [_GUARD_NOS_DICT_r12] = _GUARD_NOS_DICT, @@ -3970,6 +3986,8 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_BINARY_OP_SUBSCR_STR_INT] = "_BINARY_OP_SUBSCR_STR_INT", [_BINARY_OP_SUBSCR_STR_INT_r23] = "_BINARY_OP_SUBSCR_STR_INT_r23", [_BINARY_OP_SUBSCR_TUPLE_INT] = "_BINARY_OP_SUBSCR_TUPLE_INT", + [_BINARY_OP_SUBSCR_TUPLE_INT_r03] = "_BINARY_OP_SUBSCR_TUPLE_INT_r03", + [_BINARY_OP_SUBSCR_TUPLE_INT_r13] = "_BINARY_OP_SUBSCR_TUPLE_INT_r13", [_BINARY_OP_SUBSCR_TUPLE_INT_r23] = "_BINARY_OP_SUBSCR_TUPLE_INT_r23", [_BINARY_OP_SUBTRACT_FLOAT] = "_BINARY_OP_SUBTRACT_FLOAT", [_BINARY_OP_SUBTRACT_FLOAT_r03] = "_BINARY_OP_SUBTRACT_FLOAT_r03", @@ -4223,6 +4241,11 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GET_YIELD_FROM_ITER_r11] = "_GET_YIELD_FROM_ITER_r11", [_GUARD_BINARY_OP_EXTEND] = "_GUARD_BINARY_OP_EXTEND", [_GUARD_BINARY_OP_EXTEND_r22] = "_GUARD_BINARY_OP_EXTEND_r22", + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS] = "_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS", + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r02] = "_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r02", + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r12] = "_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r12", + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r22] = "_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r22", + [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r33] = "_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r33", [_GUARD_CALLABLE_ISINSTANCE] = "_GUARD_CALLABLE_ISINSTANCE", [_GUARD_CALLABLE_ISINSTANCE_r03] = "_GUARD_CALLABLE_ISINSTANCE_r03", [_GUARD_CALLABLE_ISINSTANCE_r13] = "_GUARD_CALLABLE_ISINSTANCE_r13", @@ -5103,6 +5126,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _GUARD_TOS_TUPLE: return 0; + case _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS: + return 0; case _BINARY_OP_SUBSCR_TUPLE_INT: return 2; case _GUARD_NOS_DICT: diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index d3c71dd8ae4449..d090b22f3bb81d 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1859,6 +1859,21 @@ def f(n): self.assertNotIn("_GUARD_NOS_TUPLE", uops) self.assertIn("_BINARY_OP_SUBSCR_TUPLE_INT", uops) + def test_remove_guard_for_tuple_bounds_check(self): + def f(n): + x = 0 + for _ in range(n): + t = (1, 2, 3) + x += t[0] + return x + + res, ex = self._run_with_optimizer(f, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS", uops) + self.assertIn("_BINARY_OP_SUBSCR_TUPLE_INT", uops) + def test_binary_subcsr_str_int_narrows_to_str(self): def testfunc(n): x = [] diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-08-10-12-46-36.gh-issue-131798.5ys0H_.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-10-12-46-36.gh-issue-131798.5ys0H_.rst new file mode 100644 index 00000000000000..5a04d143a29724 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-10-12-46-36.gh-issue-131798.5ys0H_.rst @@ -0,0 +1 @@ +Remove bounds check when indexing into tuples with a constant index. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 86a4a7f116d93f..ea09c0645aa39c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -968,9 +968,16 @@ dummy_func( } macro(BINARY_OP_SUBSCR_TUPLE_INT) = - _GUARD_TOS_INT + _GUARD_NOS_TUPLE + unused/5 + _BINARY_OP_SUBSCR_TUPLE_INT + _POP_TOP_INT + POP_TOP; + _GUARD_TOS_INT + + _GUARD_NOS_TUPLE + + _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS + + unused/5 + + _BINARY_OP_SUBSCR_TUPLE_INT + + _POP_TOP_INT + + POP_TOP; - op(_BINARY_OP_SUBSCR_TUPLE_INT, (tuple_st, sub_st -- res, ts, ss)) { + // A guard that checks that the tuple subscript is within bounds + op(_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS, (tuple_st, sub_st -- tuple_st, sub_st)) { PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); @@ -981,7 +988,17 @@ dummy_func( DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; DEOPT_IF(index >= PyTuple_GET_SIZE(tuple)); + } + + op(_BINARY_OP_SUBSCR_TUPLE_INT, (tuple_st, sub_st -- res, ts, ss)) { + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); + + assert(PyLong_CheckExact(sub)); + assert(PyTuple_CheckExact(tuple)); + STAT_INC(BINARY_OP, hit); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); res = PyStackRef_FromPyObjectNew(res_o); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 8c8a47d6e134bd..4e8fa34c0b2c0d 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4891,14 +4891,76 @@ break; } - case _BINARY_OP_SUBSCR_TUPLE_INT_r23: { + case _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef sub_st; + _PyStackRef tuple_st; + sub_st = stack_pointer[-1]; + tuple_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); + assert(PyLong_CheckExact(sub)); + assert(PyTuple_CheckExact(tuple)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (index >= PyTuple_GET_SIZE(tuple)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = sub_st; + _tos_cache0 = tuple_st; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef sub_st; + _PyStackRef tuple_st; + _PyStackRef _stack_item_0 = _tos_cache0; + sub_st = _stack_item_0; + tuple_st = stack_pointer[-1]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); + assert(PyLong_CheckExact(sub)); + assert(PyTuple_CheckExact(tuple)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = sub_st; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (index >= PyTuple_GET_SIZE(tuple)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = sub_st; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = sub_st; + _tos_cache0 = tuple_st; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r22: { CHECK_CURRENT_CACHED_VALUES(2); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef sub_st; _PyStackRef tuple_st; - _PyStackRef res; - _PyStackRef ts; - _PyStackRef ss; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; sub_st = _stack_item_1; @@ -4922,7 +4984,133 @@ SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } + _tos_cache1 = sub_st; + _tos_cache0 = tuple_st; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef sub_st; + _PyStackRef tuple_st; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + sub_st = _stack_item_2; + tuple_st = _stack_item_1; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); + assert(PyLong_CheckExact(sub)); + assert(PyTuple_CheckExact(tuple)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = sub_st; + _tos_cache1 = tuple_st; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (index >= PyTuple_GET_SIZE(tuple)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = sub_st; + _tos_cache1 = tuple_st; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = sub_st; + _tos_cache1 = tuple_st; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBSCR_TUPLE_INT_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef sub_st; + _PyStackRef tuple_st; + _PyStackRef res; + _PyStackRef ts; + _PyStackRef ss; + sub_st = stack_pointer[-1]; + tuple_st = stack_pointer[-2]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); + assert(PyLong_CheckExact(sub)); + assert(PyTuple_CheckExact(tuple)); + STAT_INC(BINARY_OP, hit); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + PyObject *res_o = PyTuple_GET_ITEM(tuple, index); + assert(res_o != NULL); + res = PyStackRef_FromPyObjectNew(res_o); + ts = tuple_st; + ss = sub_st; + _tos_cache2 = ss; + _tos_cache1 = ts; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBSCR_TUPLE_INT_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef sub_st; + _PyStackRef tuple_st; + _PyStackRef res; + _PyStackRef ts; + _PyStackRef ss; + _PyStackRef _stack_item_0 = _tos_cache0; + sub_st = _stack_item_0; + tuple_st = stack_pointer[-1]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); + assert(PyLong_CheckExact(sub)); + assert(PyTuple_CheckExact(tuple)); + STAT_INC(BINARY_OP, hit); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + PyObject *res_o = PyTuple_GET_ITEM(tuple, index); + assert(res_o != NULL); + res = PyStackRef_FromPyObjectNew(res_o); + ts = tuple_st; + ss = sub_st; + _tos_cache2 = ss; + _tos_cache1 = ts; + _tos_cache0 = res; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _BINARY_OP_SUBSCR_TUPLE_INT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef sub_st; + _PyStackRef tuple_st; + _PyStackRef res; + _PyStackRef ts; + _PyStackRef ss; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + sub_st = _stack_item_1; + tuple_st = _stack_item_0; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); + assert(PyLong_CheckExact(sub)); + assert(PyTuple_CheckExact(tuple)); STAT_INC(BINARY_OP, hit); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); res = PyStackRef_FromPyObjectNew(res_o); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 307ca1fac65e7f..e63852aee1134c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1042,8 +1042,7 @@ JUMP_TO_PREDICTED(BINARY_OP); } } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBSCR_TUPLE_INT + // _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS { sub_st = value; tuple_st = nos; @@ -1062,7 +1061,16 @@ assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_TUPLE_INT + { + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); + assert(PyLong_CheckExact(sub)); + assert(PyTuple_CheckExact(tuple)); STAT_INC(BINARY_OP, hit); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); res = PyStackRef_FromPyObjectNew(res_o); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 55680c5b824b7b..3e37cb81388ae0 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -332,6 +332,19 @@ dummy_func(void) { i = sub_st; } + op(_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS, (tuple_st, sub_st -- tuple_st, sub_st)) { + assert(sym_matches_type(tuple_st, &PyTuple_Type)); + if (sym_is_const(ctx, sub_st)) { + assert(PyLong_CheckExact(sym_get_const(ctx, sub_st))); + long index = PyLong_AsLong(sym_get_const(ctx, sub_st)); + assert(index >= 0); + int tuple_length = sym_tuple_length(tuple_st); + if (tuple_length != -1 && index < tuple_length) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + } + } + op(_BINARY_OP_SUBSCR_TUPLE_INT, (tuple_st, sub_st -- res, ts, ss)) { assert(sym_matches_type(tuple_st, &PyTuple_Type)); if (sym_is_const(ctx, sub_st)) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 5f4106c33b7f64..b043a9493cbd7c 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -799,6 +799,24 @@ break; } + case _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS: { + JitOptRef sub_st; + JitOptRef tuple_st; + sub_st = stack_pointer[-1]; + tuple_st = stack_pointer[-2]; + assert(sym_matches_type(tuple_st, &PyTuple_Type)); + if (sym_is_const(ctx, sub_st)) { + assert(PyLong_CheckExact(sym_get_const(ctx, sub_st))); + long index = PyLong_AsLong(sym_get_const(ctx, sub_st)); + assert(index >= 0); + int tuple_length = sym_tuple_length(tuple_st); + if (tuple_length != -1 && index < tuple_length) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + } + break; + } + case _BINARY_OP_SUBSCR_TUPLE_INT: { JitOptRef sub_st; JitOptRef tuple_st; From daa9aa4c0a8490f09b01339b6928434d7fe02843 Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Mon, 29 Dec 2025 06:12:31 +0800 Subject: [PATCH 604/638] gh-143183: Rewind stop tracing to previous target (GH-143187) Co-authored-by: Kumar Aditya --- Lib/test/test_capi/test_opt.py | 42 +++++++++++++++++++ ...-12-26-11-00-44.gh-issue-143183.rhxzZr.rst | 1 + Python/optimizer.c | 20 ++++----- 3 files changed, 51 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-12-26-11-00-44.gh-issue-143183.rhxzZr.rst diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index d090b22f3bb81d..ea1606fd5b5f05 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -3306,6 +3306,48 @@ class B: ... for i in range(TIER2_THRESHOLD * 10): f1() + def test_143183(self): + # https://github.com/python/cpython/issues/143183 + + result = script_helper.run_python_until_end('-c', textwrap.dedent(f""" + def f1(): + class AsyncIter: + def __init__(self): + self.limit = 0 + self.count = 0 + + def __aiter__(self): + return self + + async def __anext__(self): + if self.count >= self.limit: + ... + self.count += 1j + + class AsyncCtx: + async def async_for_driver(): + try: + for _ in range({TIER2_THRESHOLD}): + try: + async for _ in AsyncIter(): + ... + except TypeError: + ... + except Exception: + ... + + c = async_for_driver() + while True: + try: + c.send(None) + except StopIteration: + break + + for _ in range({TIER2_THRESHOLD // 40}): + f1() + """), PYTHON_JIT="1") + self.assertEqual(result[0].rc, 0, result) + def global_identity(x): return x diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-12-26-11-00-44.gh-issue-143183.rhxzZr.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-26-11-00-44.gh-issue-143183.rhxzZr.rst new file mode 100644 index 00000000000000..bee2eb672e8813 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-26-11-00-44.gh-issue-143183.rhxzZr.rst @@ -0,0 +1 @@ +Fix a bug in the JIT when dealing with unsupported control-flow or operations. diff --git a/Python/optimizer.c b/Python/optimizer.c index 5e97f20f869efd..b497ac629960ac 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -625,6 +625,7 @@ _PyJit_translate_single_bytecode_to_trace( int trace_length = _tstate->jit_tracer_state.prev_state.code_curr_size; _PyUOpInstruction *trace = _tstate->jit_tracer_state.code_buffer; int max_length = _tstate->jit_tracer_state.prev_state.code_max_size; + int exit_op = stop_tracing_opcode == 0 ? _EXIT_TRACE : stop_tracing_opcode; _Py_CODEUNIT *this_instr = _tstate->jit_tracer_state.prev_state.instr; _Py_CODEUNIT *target_instr = this_instr; @@ -691,8 +692,11 @@ _PyJit_translate_single_bytecode_to_trace( } if (stop_tracing_opcode != 0) { - ADD_TO_TRACE(stop_tracing_opcode, 0, 0, target); - goto done; + // gh-143183: It's important we rewind to the last known proper target. + // The current target might be garbage as stop tracing usually indicates + // we are in something that we can't trace. + DPRINTF(2, "Told to stop tracing\n"); + goto unsupported; } DPRINTF(2, "%p %d: %s(%d) %d %d\n", old_code, target, _PyOpcode_OpName[opcode], oparg, needs_guard_ip, old_stack_level); @@ -703,10 +707,6 @@ _PyJit_translate_single_bytecode_to_trace( } #endif - if (opcode == ENTER_EXECUTOR) { - goto full; - } - if (!_tstate->jit_tracer_state.prev_state.dependencies_still_valid) { goto full; } @@ -720,11 +720,6 @@ _PyJit_translate_single_bytecode_to_trace( if (oparg > 0xFFFF) { DPRINTF(2, "Unsupported: oparg too large\n"); - goto unsupported; - } - - // TODO (gh-140277): The constituent use one extra stack slot. So we need to check for headroom. - if (opcode == BINARY_OP_SUBSCR_GETITEM && old_stack_level + 1 > old_code->co_stacksize) { unsupported: { // Rewind to previous instruction and replace with _EXIT_TRACE. @@ -738,7 +733,7 @@ _PyJit_translate_single_bytecode_to_trace( int32_t old_target = (int32_t)uop_get_target(curr); curr++; trace_length++; - curr->opcode = _EXIT_TRACE; + curr->opcode = exit_op; curr->format = UOP_FORMAT_TARGET; curr->target = old_target; } @@ -746,6 +741,7 @@ _PyJit_translate_single_bytecode_to_trace( } } + if (opcode == NOP) { return 1; } From f37f57dfe683163f390ef589301e4dd4608c4288 Mon Sep 17 00:00:00 2001 From: Samuel Date: Mon, 29 Dec 2025 11:43:09 +0000 Subject: [PATCH 605/638] gh-131421: Fix ASDL kw_defaults being `expr*` instead of `expr?*` (GH-133773) Also fix docs ASDL highlighting. --- Doc/tools/extensions/lexers/asdl_lexer.py | 2 +- Parser/Python.asdl | 2 +- Python/Python-ast.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/tools/extensions/lexers/asdl_lexer.py b/Doc/tools/extensions/lexers/asdl_lexer.py index 3a74174a1f7dfb..548ce1271a1db8 100644 --- a/Doc/tools/extensions/lexers/asdl_lexer.py +++ b/Doc/tools/extensions/lexers/asdl_lexer.py @@ -22,7 +22,7 @@ class ASDLLexer(RegexLexer): bygroups(Keyword, Text, Name.Tag), ), ( - r"(\w+)(\*\s|\?\s|\s)(\w+)", + r"(\w+)([\?\*]*\s)(\w+)", bygroups(Name.Builtin.Pseudo, Operator, Name), ), # Keep in line with ``builtin_types`` from Parser/asdl.py. diff --git a/Parser/Python.asdl b/Parser/Python.asdl index 96f3914b029d4c..9c7529c479916d 100644 --- a/Parser/Python.asdl +++ b/Parser/Python.asdl @@ -114,7 +114,7 @@ module Python attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) arguments = (arg* posonlyargs, arg* args, arg? vararg, arg* kwonlyargs, - expr* kw_defaults, arg? kwarg, expr* defaults) + expr?* kw_defaults, arg? kwarg, expr* defaults) arg = (identifier arg, expr? annotation, string? type_comment) attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) diff --git a/Python/Python-ast.c b/Python/Python-ast.c index aac24ed7d3c0c5..79608dee9bfac2 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -6813,7 +6813,7 @@ init_types(void *arg) return -1; state->arguments_type = make_type(state, "arguments", state->AST_type, arguments_fields, 7, - "arguments(arg* posonlyargs, arg* args, arg? vararg, arg* kwonlyargs, expr* kw_defaults, arg? kwarg, expr* defaults)"); + "arguments(arg* posonlyargs, arg* args, arg? vararg, arg* kwonlyargs, expr?* kw_defaults, arg? kwarg, expr* defaults)"); if (!state->arguments_type) return -1; if (add_attributes(state, state->arguments_type, NULL, 0) < 0) return -1; if (PyObject_SetAttr(state->arguments_type, state->vararg, Py_None) == -1) From 6cb245d26086369bb075858501405865fc255a10 Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Mon, 29 Dec 2025 23:10:42 +0800 Subject: [PATCH 606/638] gh-143183: Link trace to side exits, rather than stop (GH-143268) --- Lib/test/test_capi/test_opt.py | 26 ++++++++++++++++++++++---- Modules/_testinternalcapi.c | 17 +++++++++++++++++ Python/optimizer.c | 10 +++++++--- 3 files changed, 46 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index ea1606fd5b5f05..3780bdb28c8c44 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -60,6 +60,13 @@ def iter_opnames(ex): def get_opnames(ex): return list(iter_opnames(ex)) +def iter_ops(ex): + for item in ex: + yield item + +def get_ops(ex): + return list(iter_ops(ex)) + @requires_specialization @unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds") @@ -3003,14 +3010,25 @@ def f(): # Outer loop warms up later, linking to the inner one. # Therefore, we have at least two executors. self.assertGreaterEqual(len(all_executors), 2) + executor_ids = [id(e) for e in all_executors] for executor in all_executors: - opnames = list(get_opnames(executor)) + ops = get_ops(executor) # Assert all executors first terminator ends in # _EXIT_TRACE or _JUMP_TO_TOP, not _DEOPT - for idx, op in enumerate(opnames): - if op == "_EXIT_TRACE" or op == "_JUMP_TO_TOP": + for idx, op in enumerate(ops): + opname = op[0] + if opname == "_EXIT_TRACE": + # As this is a link outer executor to inner + # executor problem, all executors exits should point to + # another valid executor. In this case, none of them + # should be the cold executor. + exit = op[3] + link_to = _testinternalcapi.get_exit_executor(exit) + self.assertIn(id(link_to), executor_ids) + break + elif opname == "_JUMP_TO_TOP": break - elif op == "_DEOPT": + elif opname == "_DEOPT": self.fail(f"_DEOPT encountered first at executor" f" {executor} at offset {idx} rather" f" than expected _EXIT_TRACE") diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index a7fbb0f87b6e9c..ea09f2d5ef836f 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1245,6 +1245,22 @@ invalidate_executors(PyObject *self, PyObject *obj) Py_RETURN_NONE; } +static PyObject * +get_exit_executor(PyObject *self, PyObject *arg) +{ + if (!PyLong_CheckExact(arg)) { + PyErr_SetString(PyExc_TypeError, "argument must be an ID to an _PyExitData"); + return NULL; + } + uint64_t ptr; + if (PyLong_AsUInt64(arg, &ptr) < 0) { + // Error set by PyLong API + return NULL; + } + _PyExitData *exit = (_PyExitData *)ptr; + return Py_NewRef(exit->executor); +} + #endif static int _pending_callback(void *arg) @@ -2546,6 +2562,7 @@ static PyMethodDef module_functions[] = { #ifdef _Py_TIER2 {"add_executor_dependency", add_executor_dependency, METH_VARARGS, NULL}, {"invalidate_executors", invalidate_executors, METH_O, NULL}, + {"get_exit_executor", get_exit_executor, METH_O, NULL}, #endif {"pending_threadfunc", _PyCFunction_CAST(pending_threadfunc), METH_VARARGS | METH_KEYWORDS}, diff --git a/Python/optimizer.c b/Python/optimizer.c index b497ac629960ac..900b07473fe351 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -625,7 +625,6 @@ _PyJit_translate_single_bytecode_to_trace( int trace_length = _tstate->jit_tracer_state.prev_state.code_curr_size; _PyUOpInstruction *trace = _tstate->jit_tracer_state.code_buffer; int max_length = _tstate->jit_tracer_state.prev_state.code_max_size; - int exit_op = stop_tracing_opcode == 0 ? _EXIT_TRACE : stop_tracing_opcode; _Py_CODEUNIT *this_instr = _tstate->jit_tracer_state.prev_state.instr; _Py_CODEUNIT *target_instr = this_instr; @@ -691,13 +690,18 @@ _PyJit_translate_single_bytecode_to_trace( goto full; } - if (stop_tracing_opcode != 0) { + if (stop_tracing_opcode == _DEOPT) { // gh-143183: It's important we rewind to the last known proper target. // The current target might be garbage as stop tracing usually indicates // we are in something that we can't trace. DPRINTF(2, "Told to stop tracing\n"); goto unsupported; } + else if (stop_tracing_opcode != 0) { + assert(stop_tracing_opcode == _EXIT_TRACE); + ADD_TO_TRACE(stop_tracing_opcode, 0, 0, target); + goto done; + } DPRINTF(2, "%p %d: %s(%d) %d %d\n", old_code, target, _PyOpcode_OpName[opcode], oparg, needs_guard_ip, old_stack_level); @@ -733,7 +737,7 @@ _PyJit_translate_single_bytecode_to_trace( int32_t old_target = (int32_t)uop_get_target(curr); curr++; trace_length++; - curr->opcode = exit_op; + curr->opcode = _DEOPT; curr->format = UOP_FORMAT_TARGET; curr->target = old_target; } From b6b0e14b3d4aa9e9b89bef9a516177238883e1a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 29 Dec 2025 18:30:51 +0100 Subject: [PATCH 607/638] gh-143200: fix UAFs in `Element.__{set,get}item__` when the element is concurrently mutated (#143226) --- Lib/test/test_xml_etree.py | 72 ++++++++++++++----- ...-12-27-15-41-27.gh-issue-143200.2hEUAl.rst | 4 ++ Modules/_elementtree.c | 40 +++++++---- 3 files changed, 85 insertions(+), 31 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-12-27-15-41-27.gh-issue-143200.2hEUAl.rst diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 0178ed02b35be1..7aa949b2819172 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -3010,32 +3010,72 @@ def __del__(self): elem = b.close() self.assertEqual(elem[0].tail, 'ABCDEFGHIJKL') - def test_subscr(self): - # Issue #27863 + def test_subscr_with_clear(self): + # See https://github.com/python/cpython/issues/143200. + self.do_test_subscr_with_mutating_slice(use_clear_method=True) + + def test_subscr_with_delete(self): + # See https://github.com/python/cpython/issues/72050. + self.do_test_subscr_with_mutating_slice(use_clear_method=False) + + def do_test_subscr_with_mutating_slice(self, *, use_clear_method): class X: + def __init__(self, i=0): + self.i = i def __index__(self): - del e[:] - return 1 + if use_clear_method: + e.clear() + else: + del e[:] + return self.i - e = ET.Element('elem') - e.append(ET.Element('child')) - e[:X()] # shouldn't crash + for s in self.get_mutating_slices(X, 10): + with self.subTest(s): + e = ET.Element('elem') + e.extend([ET.Element(f'c{i}') for i in range(10)]) + e[s] # shouldn't crash - e.append(ET.Element('child')) - e[0:10:X()] # shouldn't crash + def test_ass_subscr_with_mutating_slice(self): + # See https://github.com/python/cpython/issues/72050 + # and https://github.com/python/cpython/issues/143200. - def test_ass_subscr(self): - # Issue #27863 class X: + def __init__(self, i=0): + self.i = i def __index__(self): e[:] = [] - return 1 + return self.i + + for s in self.get_mutating_slices(X, 10): + with self.subTest(s): + e = ET.Element('elem') + e.extend([ET.Element(f'c{i}') for i in range(10)]) + e[s] = [] # shouldn't crash + + def get_mutating_slices(self, index_class, n_children): + self.assertGreaterEqual(n_children, 10) + return [ + slice(index_class(), None, None), + slice(index_class(2), None, None), + slice(None, index_class(), None), + slice(None, index_class(2), None), + slice(0, 2, index_class(1)), + slice(0, 2, index_class(2)), + slice(0, n_children, index_class(1)), + slice(0, n_children, index_class(2)), + slice(0, 2 * n_children, index_class(1)), + slice(0, 2 * n_children, index_class(2)), + ] - e = ET.Element('elem') - for _ in range(10): - e.insert(0, ET.Element('child')) + def test_ass_subscr_with_mutating_iterable_value(self): + class V: + def __iter__(self): + e.clear() + return iter([ET.Element('a'), ET.Element('b')]) - e[0:10:X()] = [] # shouldn't crash + e = ET.Element('elem') + e.extend([ET.Element(f'c{i}') for i in range(10)]) + e[:] = V() def test_treebuilder_start(self): # Issue #27863 diff --git a/Misc/NEWS.d/next/Library/2025-12-27-15-41-27.gh-issue-143200.2hEUAl.rst b/Misc/NEWS.d/next/Library/2025-12-27-15-41-27.gh-issue-143200.2hEUAl.rst new file mode 100644 index 00000000000000..8b24decf098745 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-27-15-41-27.gh-issue-143200.2hEUAl.rst @@ -0,0 +1,4 @@ +:mod:`xml.etree.ElementTree`: fix use-after-free crashes in +:meth:`~object.__getitem__` and :meth:`~object.__setitem__` methods of +:class:`~xml.etree.ElementTree.Element` when the element is concurrently +mutated. Patch by Bénédikt Tran. diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 3173b52afb31b6..22d3205e6ad314 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -1806,16 +1806,20 @@ element_subscr(PyObject *op, PyObject *item) return element_getitem(op, i); } else if (PySlice_Check(item)) { + // Note: 'slicelen' is computed once we are sure that 'self->extra' + // cannot be mutated by user-defined code. + // See https://github.com/python/cpython/issues/143200. Py_ssize_t start, stop, step, slicelen, i; size_t cur; PyObject* list; - if (!self->extra) - return PyList_New(0); - if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + + if (self->extra == NULL) { + return PyList_New(0); + } slicelen = PySlice_AdjustIndices(self->extra->length, &start, &stop, step); @@ -1858,28 +1862,26 @@ element_ass_subscr(PyObject *op, PyObject *item, PyObject *value) return element_setitem(op, i, value); } else if (PySlice_Check(item)) { + // Note: 'slicelen' is computed once we are sure that 'self->extra' + // cannot be mutated by user-defined code. + // See https://github.com/python/cpython/issues/143200. Py_ssize_t start, stop, step, slicelen, newlen, i; size_t cur; PyObject* recycle = NULL; PyObject* seq; - if (!self->extra) { - if (create_extra(self, NULL) < 0) - return -1; - } - if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } - slicelen = PySlice_AdjustIndices(self->extra->length, &start, &stop, - step); if (value == NULL) { /* Delete slice */ - size_t cur; - Py_ssize_t i; - + if (self->extra == NULL) { + return 0; + } + slicelen = PySlice_AdjustIndices(self->extra->length, &start, &stop, + step); if (slicelen <= 0) return 0; @@ -1948,8 +1950,16 @@ element_ass_subscr(PyObject *op, PyObject *item, PyObject *value) } newlen = PySequence_Fast_GET_SIZE(seq); - if (step != 1 && newlen != slicelen) - { + if (self->extra == NULL) { + if (create_extra(self, NULL) < 0) { + Py_DECREF(seq); + return -1; + } + } + slicelen = PySlice_AdjustIndices(self->extra->length, &start, &stop, + step); + + if (step != 1 && newlen != slicelen) { Py_DECREF(seq); PyErr_Format(PyExc_ValueError, "attempt to assign sequence of size %zd " From 79c03ac0015ccf1cbb759f870e2af9d68f60fe3a Mon Sep 17 00:00:00 2001 From: Thanos <111999343+Sachaa-Thanasius@users.noreply.github.com> Date: Tue, 30 Dec 2025 00:16:54 -0500 Subject: [PATCH 608/638] gh-69686: Remove untrue part of `__import__` replacement docs (#143261) Remove untrue part of `__import__` replacement docs The original statement effectively says that replacing `__import__` at global scope affects import statements, and not only that, but only import statements within the rest of the executing module. None of that has been true since at least Python 2.7, I think. This was likely missed in python/cpython#69686. --- Doc/reference/import.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index f50d02a0ef03b9..83f0ee75e7aebd 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -832,9 +832,7 @@ entirely with a custom meta path hook. If it is acceptable to only alter the behaviour of import statements without affecting other APIs that access the import system, then replacing -the builtin :func:`__import__` function may be sufficient. This technique -may also be employed at the module level to only alter the behaviour of -import statements within that module. +the builtin :func:`__import__` function may be sufficient. To selectively prevent the import of some modules from a hook early on the meta path (rather than disabling the standard import system entirely), From ef834dee89d5b9413366db4cc519b015c51b5cb9 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Tue, 30 Dec 2025 06:23:30 +0100 Subject: [PATCH 609/638] gh-128546: Document that getaddrinfo() can return raw data (#128547) Document that getaddrinfo() can return raw data This is the case for IPv6 addresses if Python was compiled with --disable-ipv6. --- Doc/library/socket.rst | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 743d768bfa1f49..b7115942d1fdd1 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -1072,10 +1072,16 @@ The :mod:`socket` module also offers various network-related services: a string representing the canonical name of the *host* if :const:`AI_CANONNAME` is part of the *flags* argument; else *canonname* will be empty. *sockaddr* is a tuple describing a socket address, whose - format depends on the returned *family* (a ``(address, port)`` 2-tuple for - :const:`AF_INET`, a ``(address, port, flowinfo, scope_id)`` 4-tuple for - :const:`AF_INET6`), and is meant to be passed to the :meth:`socket.connect` - method. + format depends on the returned *family* and flags Python was compiled with, + and is meant to be passed to the :meth:`socket.connect` method. + + *sockaddr* can be one of the following: + + * a ``(address, port)`` 2-tuple for :const:`AF_INET` + * a ``(address, port, flowinfo, scope_id)`` 4-tuple for :const:`AF_INET6` if + Python was compiled with ``--enable-ipv6`` (the default) + * a 2-tuple containing raw data for :const:`AF_INET6` if Python was + compiled with ``--disable-ipv6`` .. note:: From 23ad9c5d01d6548e9ae1c5c6edd1cd2fabc3217f Mon Sep 17 00:00:00 2001 From: dgpb <3577712+dg-pb@users.noreply.github.com> Date: Tue, 30 Dec 2025 09:15:59 +0200 Subject: [PATCH 610/638] gh-142939: difflib.get_close_matches performance (#142940) --- Lib/difflib.py | 16 +++++++++------- ...025-12-29-21-12-12.gh-issue-142939.OyQQr5.rst | 1 + 2 files changed, 10 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-12-29-21-12-12.gh-issue-142939.OyQQr5.rst diff --git a/Lib/difflib.py b/Lib/difflib.py index 4a0600e4ebb01b..7c7e233b013a76 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -638,15 +638,15 @@ def quick_ratio(self): # avail[x] is the number of times x appears in 'b' less the # number of times we've seen it in 'a' so far ... kinda avail = {} - availhas, matches = avail.__contains__, 0 + matches = 0 for elt in self.a: - if availhas(elt): + if elt in avail: numb = avail[elt] else: numb = fullbcount.get(elt, 0) avail[elt] = numb - 1 if numb > 0: - matches = matches + 1 + matches += 1 return _calculate_ratio(matches, len(self.a) + len(self.b)) def real_quick_ratio(self): @@ -702,10 +702,12 @@ def get_close_matches(word, possibilities, n=3, cutoff=0.6): s.set_seq2(word) for x in possibilities: s.set_seq1(x) - if s.real_quick_ratio() >= cutoff and \ - s.quick_ratio() >= cutoff and \ - s.ratio() >= cutoff: - result.append((s.ratio(), x)) + if s.real_quick_ratio() < cutoff or s.quick_ratio() < cutoff: + continue + + ratio = s.ratio() + if ratio >= cutoff: + result.append((ratio, x)) # Move the best scorers to head of list result = _nlargest(n, result) diff --git a/Misc/NEWS.d/next/Library/2025-12-29-21-12-12.gh-issue-142939.OyQQr5.rst b/Misc/NEWS.d/next/Library/2025-12-29-21-12-12.gh-issue-142939.OyQQr5.rst new file mode 100644 index 00000000000000..65523f08dcc0b0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-29-21-12-12.gh-issue-142939.OyQQr5.rst @@ -0,0 +1 @@ +Performance optimisations for :func:`difflib.get_close_matches` From 0aedf2f9cf896abd4e19d251ed444cdac04c7aa9 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 30 Dec 2025 14:31:41 +0200 Subject: [PATCH 611/638] gh-143284: Temporarily install Sphinx<9 to fix Chinese search (#143286) --- Doc/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/requirements.txt b/Doc/requirements.txt index 716772b7f28d99..d0107744ecbe85 100644 --- a/Doc/requirements.txt +++ b/Doc/requirements.txt @@ -7,7 +7,7 @@ # won't suddenly cause build failures. Updating the version is fine as long # as no warnings are raised by doing so. # Keep this version in sync with ``Doc/conf.py``. -sphinx~=9.0.0 +sphinx<9.0.0 blurb From 7e3a5a7e791b742a74c64810f221854191b94c1f Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 30 Dec 2025 12:57:28 +0000 Subject: [PATCH 612/638] gh-130167: Add a What's New entry for changes to ``textwrap.{de,in}dent`` (#131924) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/library/textwrap.rst | 4 ++++ Doc/whatsnew/3.14.rst | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/Doc/library/textwrap.rst b/Doc/library/textwrap.rst index a58b460fef409c..3c96c0e9cc0a38 100644 --- a/Doc/library/textwrap.rst +++ b/Doc/library/textwrap.rst @@ -102,6 +102,10 @@ functions should be good enough; otherwise, you should use an instance of print(repr(s)) # prints ' hello\n world\n ' print(repr(dedent(s))) # prints 'hello\n world\n' + .. versionchanged:: 3.14 + The :func:`!dedent` function now correctly normalizes blank lines containing + only whitespace characters. Previously, the implementation only normalized + blank lines containing tabs and spaces. .. function:: indent(text, prefix, predicate=None) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 9459b73bcb502f..c12a1920b10722 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -2265,6 +2265,15 @@ pdb (Contributed by Tian Gao in :gh:`124533`.) +textwrap +-------- + +* Optimize the :func:`~textwrap.dedent` function, improving performance by + an average of 2.4x, with larger improvements for bigger inputs, + and fix a bug with incomplete normalization of blank lines with whitespace + characters other than space and tab. + + uuid ---- From aa8a43d179bad5cd9fbfce63b630e2ee0bd617e4 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 30 Dec 2025 16:56:29 +0200 Subject: [PATCH 613/638] gh-143237: Fix support of named pipes in the rotating logging handlers (GH-143259) This fixes regression introduced in GH-105887. --- Lib/logging/handlers.py | 6 ++++- Lib/test/test_logging.py | 27 +++++++++++++++++++ ...-12-28-20-28-05.gh-issue-143237.q1ymuA.rst | 1 + 3 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-12-28-20-28-05.gh-issue-143237.q1ymuA.rst diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index 2748b5941eade2..4a07258f8d6d07 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -196,7 +196,11 @@ def shouldRollover(self, record): if self.stream is None: # delay was set... self.stream = self._open() if self.maxBytes > 0: # are we rolling over? - pos = self.stream.tell() + try: + pos = self.stream.tell() + except io.UnsupportedOperation: + # gh-143237: Never rollover a named pipe. + return False if not pos: # gh-116263: Never rollover an empty file return False diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 8815426fc99c39..848084e6e36878 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -25,6 +25,7 @@ import codecs import configparser +import contextlib import copy import datetime import pathlib @@ -6369,6 +6370,32 @@ def test_should_not_rollover_non_file(self): self.assertFalse(rh.shouldRollover(self.next_rec())) rh.close() + @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') + def test_should_not_rollover_named_pipe(self): + # gh-143237 - test with non-seekable special file (named pipe) + filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, filename) + try: + os.mkfifo(filename) + except PermissionError as e: + self.skipTest('os.mkfifo(): %s' % e) + + data = 'not read' + def other_side(): + nonlocal data + with open(filename, 'rb') as f: + data = f.read() + + thread = threading.Thread(target=other_side) + with threading_helper.start_threads([thread]): + rh = logging.handlers.RotatingFileHandler( + filename, encoding="utf-8", maxBytes=1) + with contextlib.closing(rh): + m = self.next_rec() + self.assertFalse(rh.shouldRollover(m)) + rh.emit(m) + self.assertEqual(data.decode(), m.msg + os.linesep) + def test_should_rollover(self): with open(self.fn, 'wb') as f: f.write(b'\n') diff --git a/Misc/NEWS.d/next/Library/2025-12-28-20-28-05.gh-issue-143237.q1ymuA.rst b/Misc/NEWS.d/next/Library/2025-12-28-20-28-05.gh-issue-143237.q1ymuA.rst new file mode 100644 index 00000000000000..131bebcd984ac2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-28-20-28-05.gh-issue-143237.q1ymuA.rst @@ -0,0 +1 @@ +Fix support of named pipes in the rotating :mod:`logging` handlers. From 04899b8539ab83657a4495203f26b3cb1a6f46dc Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" <68491+gpshead@users.noreply.github.com> Date: Tue, 30 Dec 2025 15:24:32 -0800 Subject: [PATCH 614/638] gh-115634: document ProcessPoolExecutor max_tasks_per_child bug (GH-140897) --- Doc/library/concurrent.futures.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index c2e2f7f820f4ef..18d92e8e9959a9 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -379,6 +379,11 @@ in a REPL or a lambda should not be expected to work. default in absence of a *mp_context* parameter. This feature is incompatible with the "fork" start method. + .. note:: + Bugs have been reported when using the *max_tasks_per_child* feature that + can result in the :class:`ProcessPoolExecutor` hanging in some + circumstances. Follow its eventual resolution in :gh:`115634`. + .. versionchanged:: 3.3 When one of the worker processes terminates abruptly, a :exc:`~concurrent.futures.process.BrokenProcessPool` error is now raised. From 469fe33edd92b8586d6995d07384b52170067c76 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Tue, 30 Dec 2025 19:45:23 -0500 Subject: [PATCH 615/638] gh-143121: Avoid thread leak in configure (gh-143122) If you are building with `--with-thread-sanitizer` and don't use the suppression file, then running configure will report a thread leak. Call `pthread_join()` to avoid the thread leak. --- configure | 1 + configure.ac | 1 + 2 files changed, 2 insertions(+) diff --git a/configure b/configure index b1faeaf806a9c6..411bc1a23226e7 100755 --- a/configure +++ b/configure @@ -18190,6 +18190,7 @@ else case e in #( if (pthread_attr_init(&attr)) return (-1); if (pthread_attr_setscope(&attr, PTHREAD_SCOPE_SYSTEM)) return (-1); if (pthread_create(&id, &attr, foo, NULL)) return (-1); + if (pthread_join(id, NULL)) return (-1); return (0); } _ACEOF diff --git a/configure.ac b/configure.ac index 043ec957f40894..9e63c8f6144c3d 100644 --- a/configure.ac +++ b/configure.ac @@ -4760,6 +4760,7 @@ if test "$posix_threads" = "yes"; then if (pthread_attr_init(&attr)) return (-1); if (pthread_attr_setscope(&attr, PTHREAD_SCOPE_SYSTEM)) return (-1); if (pthread_create(&id, &attr, foo, NULL)) return (-1); + if (pthread_join(id, NULL)) return (-1); return (0); }]])], [ac_cv_pthread_system_supported=yes], From 96ab379dcaa93630a230402b8183a26ac99097bd Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Tue, 30 Dec 2025 19:45:44 -0500 Subject: [PATCH 616/638] gh-140795: Keep 'err' in local variable in _ssl.c (gh-143275) The error return code doesn't need to be mutable state on the SSLSocket. This simplifes thread-safety and avoids potential reentrancy issues. --- Modules/_ssl.c | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 25fcea6aaf128d..5d2f075ed0c675 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -360,7 +360,6 @@ typedef struct { enum py_ssl_server_or_client socket_type; PyObject *owner; /* Python level "owner" passed to servername callback */ PyObject *server_hostname; - _PySSLError err; /* last seen error from various sources */ /* Some SSL callbacks don't have error reporting. Callback wrappers * store exception information on the socket. The handshake, read, write, * and shutdown methods check for chained exceptions. @@ -669,11 +668,10 @@ PySSL_ChainExceptions(PySSLSocket *sslsock) { } static PyObject * -PySSL_SetError(PySSLSocket *sslsock, const char *filename, int lineno) +PySSL_SetError(PySSLSocket *sslsock, _PySSLError err, const char *filename, int lineno) { PyObject *type; char *errstr = NULL; - _PySSLError err; enum py_ssl_error p = PY_SSL_ERROR_NONE; unsigned long e = 0; @@ -686,8 +684,6 @@ PySSL_SetError(PySSLSocket *sslsock, const char *filename, int lineno) e = ERR_peek_last_error(); if (sslsock->ssl != NULL) { - err = sslsock->err; - switch (err.ssl) { case SSL_ERROR_ZERO_RETURN: errstr = "TLS/SSL connection has been closed (EOF)"; @@ -885,7 +881,6 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, { PySSLSocket *self; SSL_CTX *ctx = sslctx->ctx; - _PySSLError err = { 0 }; if ((socket_type == PY_SSL_SERVER) && (sslctx->protocol == PY_SSL_VERSION_TLS_CLIENT)) { @@ -913,7 +908,6 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, self->shutdown_seen_zero = 0; self->owner = NULL; self->server_hostname = NULL; - self->err = err; self->exc = NULL; /* Make sure the SSL error state is initialized */ @@ -1069,7 +1063,6 @@ _ssl__SSLSocket_do_handshake_impl(PySSLSocket *self) err = _PySSL_errno(ret < 1, self->ssl, ret); Py_END_ALLOW_THREADS; _PySSL_FIX_ERRNO; - self->err = err; if (PyErr_CheckSignals()) goto error; @@ -1105,7 +1098,7 @@ _ssl__SSLSocket_do_handshake_impl(PySSLSocket *self) Py_XDECREF(sock); if (ret < 1) - return PySSL_SetError(self, __FILE__, __LINE__); + return PySSL_SetError(self, err, __FILE__, __LINE__); if (PySSL_ChainExceptions(self) < 0) return NULL; Py_RETURN_NONE; @@ -2672,7 +2665,6 @@ _ssl__SSLSocket_sendfile_impl(PySSLSocket *self, int fd, Py_off_t offset, err = _PySSL_errno(retval < 0, self->ssl, (int)retval); Py_END_ALLOW_THREADS; _PySSL_FIX_ERRNO; - self->err = err; if (PyErr_CheckSignals()) { goto error; @@ -2723,7 +2715,7 @@ _ssl__SSLSocket_sendfile_impl(PySSLSocket *self, int fd, Py_off_t offset, } Py_XDECREF(sock); if (retval < 0) { - return PySSL_SetError(self, __FILE__, __LINE__); + return PySSL_SetError(self, err, __FILE__, __LINE__); } if (PySSL_ChainExceptions(self) < 0) { return NULL; @@ -2804,7 +2796,6 @@ _ssl__SSLSocket_write_impl(PySSLSocket *self, Py_buffer *b) err = _PySSL_errno(retval == 0, self->ssl, retval); Py_END_ALLOW_THREADS; _PySSL_FIX_ERRNO; - self->err = err; if (PyErr_CheckSignals()) goto error; @@ -2837,7 +2828,7 @@ _ssl__SSLSocket_write_impl(PySSLSocket *self, Py_buffer *b) Py_XDECREF(sock); if (retval == 0) - return PySSL_SetError(self, __FILE__, __LINE__); + return PySSL_SetError(self, err, __FILE__, __LINE__); if (PySSL_ChainExceptions(self) < 0) return NULL; return PyLong_FromSize_t(count); @@ -2867,10 +2858,9 @@ _ssl__SSLSocket_pending_impl(PySSLSocket *self) err = _PySSL_errno(count < 0, self->ssl, count); Py_END_ALLOW_THREADS; _PySSL_FIX_ERRNO; - self->err = err; if (count < 0) - return PySSL_SetError(self, __FILE__, __LINE__); + return PySSL_SetError(self, err, __FILE__, __LINE__); else return PyLong_FromLong(count); } @@ -2964,7 +2954,6 @@ _ssl__SSLSocket_read_impl(PySSLSocket *self, Py_ssize_t len, err = _PySSL_errno(retval == 0, self->ssl, retval); Py_END_ALLOW_THREADS; _PySSL_FIX_ERRNO; - self->err = err; if (PyErr_CheckSignals()) goto error; @@ -2997,7 +2986,7 @@ _ssl__SSLSocket_read_impl(PySSLSocket *self, Py_ssize_t len, err.ssl == SSL_ERROR_WANT_WRITE); if (retval == 0) { - PySSL_SetError(self, __FILE__, __LINE__); + PySSL_SetError(self, err, __FILE__, __LINE__); goto error; } if (self->exc != NULL) @@ -3077,7 +3066,6 @@ _ssl__SSLSocket_shutdown_impl(PySSLSocket *self) err = _PySSL_errno(ret < 0, self->ssl, ret); Py_END_ALLOW_THREADS; _PySSL_FIX_ERRNO; - self->err = err; /* If err == 1, a secure shutdown with SSL_shutdown() is complete */ if (ret > 0) @@ -3125,7 +3113,7 @@ _ssl__SSLSocket_shutdown_impl(PySSLSocket *self) } if (ret < 0) { Py_XDECREF(sock); - PySSL_SetError(self, __FILE__, __LINE__); + PySSL_SetError(self, err, __FILE__, __LINE__); return NULL; } if (self->exc != NULL) From 3c4429f65a894af2a0aea2aeed5f61bc399e5af5 Mon Sep 17 00:00:00 2001 From: AN Long Date: Wed, 31 Dec 2025 19:50:50 +0900 Subject: [PATCH 617/638] gh-135852: Remove out of tree pywin32 dependency for NTEventLogHandler (GH-137860) Add RegisterEventSource(), DeregisterEventSource(), ReportEvent() and a number of EVENTLOG_* constants to _winapi. --- Lib/logging/handlers.py | 91 ++++++----- Lib/test/test_logging.py | 13 +- Lib/test/test_winapi.py | 36 +++++ ...-08-17-00-28-50.gh-issue-135852.lQqOjQ.rst | 4 + Modules/_winapi.c | 106 +++++++++++++ Modules/clinic/_winapi.c.h | 149 +++++++++++++++++- 6 files changed, 358 insertions(+), 41 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-08-17-00-28-50.gh-issue-135852.lQqOjQ.rst diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index 4a07258f8d6d07..575f2babbc4785 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -1129,7 +1129,7 @@ class NTEventLogHandler(logging.Handler): """ A handler class which sends events to the NT Event Log. Adds a registry entry for the specified application name. If no dllname is - provided, win32service.pyd (which contains some basic message + provided and pywin32 installed, win32service.pyd (which contains some basic message placeholders) is used. Note that use of these placeholders will make your event logs big, as the entire message source is held in the log. If you want slimmer logs, you have to pass in the name of your own DLL @@ -1137,38 +1137,46 @@ class NTEventLogHandler(logging.Handler): """ def __init__(self, appname, dllname=None, logtype="Application"): logging.Handler.__init__(self) - try: - import win32evtlogutil, win32evtlog - self.appname = appname - self._welu = win32evtlogutil - if not dllname: - dllname = os.path.split(self._welu.__file__) + import _winapi + self._winapi = _winapi + self.appname = appname + if not dllname: + # backward compatibility + try: + import win32evtlogutil + dllname = os.path.split(win32evtlogutil.__file__) dllname = os.path.split(dllname[0]) dllname = os.path.join(dllname[0], r'win32service.pyd') - self.dllname = dllname - self.logtype = logtype - # Administrative privileges are required to add a source to the registry. - # This may not be available for a user that just wants to add to an - # existing source - handle this specific case. - try: - self._welu.AddSourceToRegistry(appname, dllname, logtype) - except Exception as e: - # This will probably be a pywintypes.error. Only raise if it's not - # an "access denied" error, else let it pass - if getattr(e, 'winerror', None) != 5: # not access denied - raise - self.deftype = win32evtlog.EVENTLOG_ERROR_TYPE - self.typemap = { - logging.DEBUG : win32evtlog.EVENTLOG_INFORMATION_TYPE, - logging.INFO : win32evtlog.EVENTLOG_INFORMATION_TYPE, - logging.WARNING : win32evtlog.EVENTLOG_WARNING_TYPE, - logging.ERROR : win32evtlog.EVENTLOG_ERROR_TYPE, - logging.CRITICAL: win32evtlog.EVENTLOG_ERROR_TYPE, - } - except ImportError: - print("The Python Win32 extensions for NT (service, event "\ - "logging) appear not to be available.") - self._welu = None + except ImportError: + pass + self.dllname = dllname + self.logtype = logtype + # Administrative privileges are required to add a source to the registry. + # This may not be available for a user that just wants to add to an + # existing source - handle this specific case. + try: + self._add_source_to_registry(appname, dllname, logtype) + except PermissionError: + pass + self.deftype = _winapi.EVENTLOG_ERROR_TYPE + self.typemap = { + logging.DEBUG: _winapi.EVENTLOG_INFORMATION_TYPE, + logging.INFO: _winapi.EVENTLOG_INFORMATION_TYPE, + logging.WARNING: _winapi.EVENTLOG_WARNING_TYPE, + logging.ERROR: _winapi.EVENTLOG_ERROR_TYPE, + logging.CRITICAL: _winapi.EVENTLOG_ERROR_TYPE, + } + + @staticmethod + def _add_source_to_registry(appname, dllname, logtype): + import winreg + + key_path = f"SYSTEM\\CurrentControlSet\\Services\\EventLog\\{logtype}\\{appname}" + + with winreg.CreateKey(winreg.HKEY_LOCAL_MACHINE, key_path) as key: + if dllname: + winreg.SetValueEx(key, "EventMessageFile", 0, winreg.REG_EXPAND_SZ, dllname) + winreg.SetValueEx(key, "TypesSupported", 0, winreg.REG_DWORD, 7) # All types are supported def getMessageID(self, record): """ @@ -1209,15 +1217,20 @@ def emit(self, record): Determine the message ID, event category and event type. Then log the message in the NT event log. """ - if self._welu: + try: + id = self.getMessageID(record) + cat = self.getEventCategory(record) + type = self.getEventType(record) + msg = self.format(record) + + # Get a handle to the event log + handle = self._winapi.RegisterEventSource(None, self.appname) try: - id = self.getMessageID(record) - cat = self.getEventCategory(record) - type = self.getEventType(record) - msg = self.format(record) - self._welu.ReportEvent(self.appname, id, cat, type, [msg]) - except Exception: - self.handleError(record) + self._winapi.ReportEvent(handle, type, cat, id, msg) + finally: + self._winapi.DeregisterEventSource(handle) + except Exception: + self.handleError(record) def close(self): """ diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 848084e6e36878..05dcea6ce0e98a 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -7248,8 +7248,8 @@ def test_compute_rollover(self, when=when, interval=interval, exp=exp): setattr(TimedRotatingFileHandlerTest, name, test_compute_rollover) -@unittest.skipUnless(win32evtlog, 'win32evtlog/win32evtlogutil/pywintypes required for this test.') class NTEventLogHandlerTest(BaseTest): + @unittest.skipUnless(win32evtlog, 'win32evtlog/win32evtlogutil/pywintypes required for this test.') def test_basic(self): logtype = 'Application' elh = win32evtlog.OpenEventLog(None, logtype) @@ -7283,6 +7283,17 @@ def test_basic(self): msg = 'Record not found in event log, went back %d records' % GO_BACK self.assertTrue(found, msg=msg) + @unittest.skipUnless(sys.platform == "win32", "Windows required for this test") + def test_without_pywin32(self): + h = logging.handlers.NTEventLogHandler('python_test') + self.addCleanup(h.close) + + # Verify that the handler uses _winapi module + self.assertIsNotNone(h._winapi, "_winapi module should be available") + + r = logging.makeLogRecord({'msg': 'Hello!'}) + h.emit(r) + class MiscTestCase(unittest.TestCase): def test__all__(self): diff --git a/Lib/test/test_winapi.py b/Lib/test/test_winapi.py index e64208330ad2f9..a1c0b80d47e4d4 100644 --- a/Lib/test/test_winapi.py +++ b/Lib/test/test_winapi.py @@ -1,5 +1,6 @@ # Test the Windows-only _winapi module +import errno import os import pathlib import re @@ -156,3 +157,38 @@ def test_namedpipe(self): pipe2.write(b'testdata') pipe2.flush() self.assertEqual((b'testdata', 8), _winapi.PeekNamedPipe(pipe, 8)[:2]) + + def test_event_source_registration(self): + source_name = "PythonTestEventSource" + + handle = _winapi.RegisterEventSource(None, source_name) + self.addCleanup(_winapi.DeregisterEventSource, handle) + self.assertNotEqual(handle, _winapi.INVALID_HANDLE_VALUE) + + with self.assertRaises(OSError) as cm: + _winapi.RegisterEventSource(None, "") + self.assertEqual(cm.exception.errno, errno.EINVAL) + + with self.assertRaises(OSError) as cm: + _winapi.DeregisterEventSource(_winapi.INVALID_HANDLE_VALUE) + self.assertEqual(cm.exception.errno, errno.EBADF) + + def test_report_event(self): + source_name = "PythonTestEventSource" + + handle = _winapi.RegisterEventSource(None, source_name) + self.assertNotEqual(handle, _winapi.INVALID_HANDLE_VALUE) + self.addCleanup(_winapi.DeregisterEventSource, handle) + + _winapi.ReportEvent(handle, _winapi.EVENTLOG_SUCCESS, 1, 1002, + "Test message 1") + + with self.assertRaises(TypeError): + _winapi.ReportEvent(handle, _winapi.EVENTLOG_SUCCESS, 1, 1002, 42) + + with self.assertRaises(TypeError): + _winapi.ReportEvent(handle, _winapi.EVENTLOG_SUCCESS, 1, 1002, None) + + with self.assertRaises(ValueError): + _winapi.ReportEvent(handle, _winapi.EVENTLOG_SUCCESS, 1, 1002, + "Test message \0 with embedded null character") diff --git a/Misc/NEWS.d/next/Library/2025-08-17-00-28-50.gh-issue-135852.lQqOjQ.rst b/Misc/NEWS.d/next/Library/2025-08-17-00-28-50.gh-issue-135852.lQqOjQ.rst new file mode 100644 index 00000000000000..7527cac330d5c4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-08-17-00-28-50.gh-issue-135852.lQqOjQ.rst @@ -0,0 +1,4 @@ +Add :func:`!_winapi.RegisterEventSource`, +:func:`!_winapi.DeregisterEventSource` and :func:`!_winapi.ReportEvent`. +Using these functions in :class:`~logging.handlers.NTEventLogHandler` +to replace :mod:`!pywin32`. diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 2aebe44c70921d..ca16b06f83010a 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -2982,6 +2982,103 @@ _winapi_CopyFile2_impl(PyObject *module, LPCWSTR existing_file_name, Py_RETURN_NONE; } +/*[clinic input] +_winapi.RegisterEventSource -> HANDLE + + unc_server_name: LPCWSTR(accept={str, NoneType}) + The UNC name of the server on which the event source should be registered. + If None, registers the event source on the local computer. + source_name: LPCWSTR + The name of the event source to register. + / + +Retrieves a registered handle to the specified event log. +[clinic start generated code]*/ + +static HANDLE +_winapi_RegisterEventSource_impl(PyObject *module, LPCWSTR unc_server_name, + LPCWSTR source_name) +/*[clinic end generated code: output=e376c8950a89ae8f input=9d01059ac2156d0c]*/ +{ + HANDLE handle; + + Py_BEGIN_ALLOW_THREADS + handle = RegisterEventSourceW(unc_server_name, source_name); + Py_END_ALLOW_THREADS + + if (handle == NULL) { + PyErr_SetFromWindowsErr(0); + return INVALID_HANDLE_VALUE; + } + + return handle; +} + +/*[clinic input] +_winapi.DeregisterEventSource + + handle: HANDLE + The handle to the event log to be deregistered. + / + +Closes the specified event log. +[clinic start generated code]*/ + +static PyObject * +_winapi_DeregisterEventSource_impl(PyObject *module, HANDLE handle) +/*[clinic end generated code: output=7387ff34c7358bce input=947593cf67641f16]*/ +{ + BOOL success; + + Py_BEGIN_ALLOW_THREADS + success = DeregisterEventSource(handle); + Py_END_ALLOW_THREADS + + if (!success) { + return PyErr_SetFromWindowsErr(0); + } + + Py_RETURN_NONE; +} + +/*[clinic input] +_winapi.ReportEvent + + handle: HANDLE + The handle to the event log. + type: unsigned_short(bitwise=False) + The type of event being reported. + category: unsigned_short(bitwise=False) + The event category. + event_id: unsigned_int(bitwise=False) + The event identifier. + string: LPCWSTR + A string to be inserted into the event message. + / + +Writes an entry at the end of the specified event log. +[clinic start generated code]*/ + +static PyObject * +_winapi_ReportEvent_impl(PyObject *module, HANDLE handle, + unsigned short type, unsigned short category, + unsigned int event_id, LPCWSTR string) +/*[clinic end generated code: output=4281230b70a2470a input=8fb3385b8e7a6d3d]*/ +{ + BOOL success; + + Py_BEGIN_ALLOW_THREADS + success = ReportEventW(handle, type, category, event_id, NULL, 1, 0, + &string, NULL); + Py_END_ALLOW_THREADS + + if (!success) { + return PyErr_SetFromWindowsErr(0); + } + + Py_RETURN_NONE; +} + static PyMethodDef winapi_functions[] = { _WINAPI_CLOSEHANDLE_METHODDEF @@ -2994,6 +3091,7 @@ static PyMethodDef winapi_functions[] = { _WINAPI_CREATEPIPE_METHODDEF _WINAPI_CREATEPROCESS_METHODDEF _WINAPI_CREATEJUNCTION_METHODDEF + _WINAPI_DEREGISTEREVENTSOURCE_METHODDEF _WINAPI_DUPLICATEHANDLE_METHODDEF _WINAPI_EXITPROCESS_METHODDEF _WINAPI_GETCURRENTPROCESS_METHODDEF @@ -3010,6 +3108,8 @@ static PyMethodDef winapi_functions[] = { _WINAPI_OPENMUTEXW_METHODDEF _WINAPI_OPENPROCESS_METHODDEF _WINAPI_PEEKNAMEDPIPE_METHODDEF + _WINAPI_REGISTEREVENTSOURCE_METHODDEF + _WINAPI_REPORTEVENT_METHODDEF _WINAPI_LCMAPSTRINGEX_METHODDEF _WINAPI_READFILE_METHODDEF _WINAPI_RELEASEMUTEX_METHODDEF @@ -3082,6 +3182,12 @@ static int winapi_exec(PyObject *m) WINAPI_CONSTANT(F_DWORD, ERROR_PIPE_CONNECTED); WINAPI_CONSTANT(F_DWORD, ERROR_PRIVILEGE_NOT_HELD); WINAPI_CONSTANT(F_DWORD, ERROR_SEM_TIMEOUT); + WINAPI_CONSTANT(F_DWORD, EVENTLOG_SUCCESS); + WINAPI_CONSTANT(F_DWORD, EVENTLOG_AUDIT_FAILURE); + WINAPI_CONSTANT(F_DWORD, EVENTLOG_AUDIT_SUCCESS); + WINAPI_CONSTANT(F_DWORD, EVENTLOG_ERROR_TYPE); + WINAPI_CONSTANT(F_DWORD, EVENTLOG_INFORMATION_TYPE); + WINAPI_CONSTANT(F_DWORD, EVENTLOG_WARNING_TYPE); WINAPI_CONSTANT(F_DWORD, FILE_FLAG_FIRST_PIPE_INSTANCE); WINAPI_CONSTANT(F_DWORD, FILE_FLAG_OVERLAPPED); WINAPI_CONSTANT(F_DWORD, FILE_GENERIC_READ); diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h index bd685e75d9344f..00cce91dca43b1 100644 --- a/Modules/clinic/_winapi.c.h +++ b/Modules/clinic/_winapi.c.h @@ -2184,7 +2184,154 @@ _winapi_CopyFile2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO return return_value; } +PyDoc_STRVAR(_winapi_RegisterEventSource__doc__, +"RegisterEventSource($module, unc_server_name, source_name, /)\n" +"--\n" +"\n" +"Retrieves a registered handle to the specified event log.\n" +"\n" +" unc_server_name\n" +" The UNC name of the server on which the event source should be registered.\n" +" If None, registers the event source on the local computer.\n" +" source_name\n" +" The name of the event source to register."); + +#define _WINAPI_REGISTEREVENTSOURCE_METHODDEF \ + {"RegisterEventSource", _PyCFunction_CAST(_winapi_RegisterEventSource), METH_FASTCALL, _winapi_RegisterEventSource__doc__}, + +static HANDLE +_winapi_RegisterEventSource_impl(PyObject *module, LPCWSTR unc_server_name, + LPCWSTR source_name); + +static PyObject * +_winapi_RegisterEventSource(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + LPCWSTR unc_server_name = NULL; + LPCWSTR source_name = NULL; + HANDLE _return_value; + + if (!_PyArg_CheckPositional("RegisterEventSource", nargs, 2, 2)) { + goto exit; + } + if (args[0] == Py_None) { + unc_server_name = NULL; + } + else if (PyUnicode_Check(args[0])) { + unc_server_name = PyUnicode_AsWideCharString(args[0], NULL); + if (unc_server_name == NULL) { + goto exit; + } + } + else { + _PyArg_BadArgument("RegisterEventSource", "argument 1", "str or None", args[0]); + goto exit; + } + if (!PyUnicode_Check(args[1])) { + _PyArg_BadArgument("RegisterEventSource", "argument 2", "str", args[1]); + goto exit; + } + source_name = PyUnicode_AsWideCharString(args[1], NULL); + if (source_name == NULL) { + goto exit; + } + _return_value = _winapi_RegisterEventSource_impl(module, unc_server_name, source_name); + if ((_return_value == INVALID_HANDLE_VALUE) && PyErr_Occurred()) { + goto exit; + } + if (_return_value == NULL) { + Py_RETURN_NONE; + } + return_value = HANDLE_TO_PYNUM(_return_value); + +exit: + /* Cleanup for unc_server_name */ + PyMem_Free((void *)unc_server_name); + /* Cleanup for source_name */ + PyMem_Free((void *)source_name); + + return return_value; +} + +PyDoc_STRVAR(_winapi_DeregisterEventSource__doc__, +"DeregisterEventSource($module, handle, /)\n" +"--\n" +"\n" +"Closes the specified event log.\n" +"\n" +" handle\n" +" The handle to the event log to be deregistered."); + +#define _WINAPI_DEREGISTEREVENTSOURCE_METHODDEF \ + {"DeregisterEventSource", (PyCFunction)_winapi_DeregisterEventSource, METH_O, _winapi_DeregisterEventSource__doc__}, + +static PyObject * +_winapi_DeregisterEventSource_impl(PyObject *module, HANDLE handle); + +static PyObject * +_winapi_DeregisterEventSource(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + HANDLE handle; + + if (!PyArg_Parse(arg, "" F_HANDLE ":DeregisterEventSource", &handle)) { + goto exit; + } + return_value = _winapi_DeregisterEventSource_impl(module, handle); + +exit: + return return_value; +} + +PyDoc_STRVAR(_winapi_ReportEvent__doc__, +"ReportEvent($module, handle, type, category, event_id, string, /)\n" +"--\n" +"\n" +"Writes an entry at the end of the specified event log.\n" +"\n" +" handle\n" +" The handle to the event log.\n" +" type\n" +" The type of event being reported.\n" +" category\n" +" The event category.\n" +" event_id\n" +" The event identifier.\n" +" string\n" +" A string to be inserted into the event message."); + +#define _WINAPI_REPORTEVENT_METHODDEF \ + {"ReportEvent", _PyCFunction_CAST(_winapi_ReportEvent), METH_FASTCALL, _winapi_ReportEvent__doc__}, + +static PyObject * +_winapi_ReportEvent_impl(PyObject *module, HANDLE handle, + unsigned short type, unsigned short category, + unsigned int event_id, LPCWSTR string); + +static PyObject * +_winapi_ReportEvent(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + HANDLE handle; + unsigned short type; + unsigned short category; + unsigned int event_id; + LPCWSTR string = NULL; + + if (!_PyArg_ParseStack(args, nargs, "" F_HANDLE "O&O&O&O&:ReportEvent", + &handle, _PyLong_UnsignedShort_Converter, &type, _PyLong_UnsignedShort_Converter, &category, _PyLong_UnsignedInt_Converter, &event_id, _PyUnicode_WideCharString_Converter, &string)) { + goto exit; + } + return_value = _winapi_ReportEvent_impl(module, handle, type, category, event_id, string); + +exit: + /* Cleanup for string */ + PyMem_Free((void *)string); + + return return_value; +} + #ifndef _WINAPI_GETSHORTPATHNAME_METHODDEF #define _WINAPI_GETSHORTPATHNAME_METHODDEF #endif /* !defined(_WINAPI_GETSHORTPATHNAME_METHODDEF) */ -/*[clinic end generated code: output=4581fd481c3c6293 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=4ab94eaee93a0a90 input=a9049054013a1b77]*/ From c5215978ebfea9471f313d5baa70a4e68bfb798b Mon Sep 17 00:00:00 2001 From: Lakshya Upadhyaya Date: Thu, 1 Jan 2026 02:15:41 +0530 Subject: [PATCH 618/638] gh-140920: remove incorrect mentions to `concurrent.futures.interpreter.ExecutionFailed` (#141723) Remove documentation for inexistant `concurrent.futures.interpreter.ExecutionFailed` and replace its occurrences by `concurrent.interpreters.ExecutionFailed` since this is the documented exception. --- Doc/library/concurrent.futures.rst | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index 18d92e8e9959a9..8b6d749f40cbf0 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -308,7 +308,7 @@ the bytes over a shared :mod:`socket ` or .. note:: The executor may replace uncaught exceptions from *initializer* - with :class:`~concurrent.futures.interpreter.ExecutionFailed`. + with :class:`~concurrent.interpreters.ExecutionFailed`. Other caveats from parent :class:`ThreadPoolExecutor` apply here. @@ -320,11 +320,11 @@ likewise serializes the return value when sending it back. When a worker's current task raises an uncaught exception, the worker always tries to preserve the exception as-is. If that is successful then it also sets the ``__cause__`` to a corresponding -:class:`~concurrent.futures.interpreter.ExecutionFailed` +:class:`~concurrent.interpreters.ExecutionFailed` instance, which contains a summary of the original exception. In the uncommon case that the worker is not able to preserve the original as-is then it directly preserves the corresponding -:class:`~concurrent.futures.interpreter.ExecutionFailed` +:class:`~concurrent.interpreters.ExecutionFailed` instance instead. @@ -720,15 +720,6 @@ Exception classes .. versionadded:: 3.14 -.. exception:: ExecutionFailed - - Raised from :class:`~concurrent.futures.InterpreterPoolExecutor` when - the given initializer fails or from - :meth:`~concurrent.futures.Executor.submit` when there's an uncaught - exception from the submitted task. - - .. versionadded:: 3.14 - .. currentmodule:: concurrent.futures.process .. exception:: BrokenProcessPool From 7f6c16a956d598663d8c67071c492f197045d967 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 1 Jan 2026 11:55:05 +0100 Subject: [PATCH 619/638] gh-142830: prevent some crashes when mutating `sqlite3` callbacks (#143245) --- Lib/test/test_sqlite3/test_hooks.py | 121 +++++++++++++++++- ...-12-28-13-12-40.gh-issue-142830.uEyd6r.rst | 2 + Modules/_sqlite/connection.c | 88 +++++++++---- Modules/_sqlite/connection.h | 1 + 4 files changed, 184 insertions(+), 28 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-12-28-13-12-40.gh-issue-142830.uEyd6r.rst diff --git a/Lib/test/test_sqlite3/test_hooks.py b/Lib/test/test_sqlite3/test_hooks.py index 2b907e35131d06..495ef97fa3c61c 100644 --- a/Lib/test/test_sqlite3/test_hooks.py +++ b/Lib/test/test_sqlite3/test_hooks.py @@ -24,11 +24,15 @@ import sqlite3 as sqlite import unittest +from test.support import import_helper from test.support.os_helper import TESTFN, unlink from .util import memory_database, cx_limit, with_tracebacks from .util import MemoryDatabaseMixin +# TODO(picnixz): increase test coverage for other callbacks +# such as 'func', 'step', 'finalize', and 'collation'. + class CollationTests(MemoryDatabaseMixin, unittest.TestCase): @@ -129,8 +133,55 @@ def test_deregister_collation(self): self.assertEqual(str(cm.exception), 'no such collation sequence: mycoll') +class AuthorizerTests(MemoryDatabaseMixin, unittest.TestCase): + + def assert_not_authorized(self, func, /, *args, **kwargs): + with self.assertRaisesRegex(sqlite.DatabaseError, "not authorized"): + func(*args, **kwargs) + + # When a handler has an invalid signature, the exception raised is + # the same that would be raised if the handler "negatively" replied. + + def test_authorizer_invalid_signature(self): + self.cx.execute("create table if not exists test(a number)") + self.cx.set_authorizer(lambda: None) + self.assert_not_authorized(self.cx.execute, "select * from test") + + # Tests for checking that callback context mutations do not crash. + # Regression tests for https://github.com/python/cpython/issues/142830. + + @with_tracebacks(ZeroDivisionError, regex="hello world") + def test_authorizer_concurrent_mutation_in_call(self): + self.cx.execute("create table if not exists test(a number)") + + def handler(*a, **kw): + self.cx.set_authorizer(None) + raise ZeroDivisionError("hello world") + + self.cx.set_authorizer(handler) + self.assert_not_authorized(self.cx.execute, "select * from test") + + @with_tracebacks(OverflowError) + def test_authorizer_concurrent_mutation_with_overflown_value(self): + _testcapi = import_helper.import_module("_testcapi") + self.cx.execute("create table if not exists test(a number)") + + def handler(*a, **kw): + self.cx.set_authorizer(None) + # We expect 'int' at the C level, so this one will raise + # when converting via PyLong_Int(). + return _testcapi.INT_MAX + 1 + + self.cx.set_authorizer(handler) + self.assert_not_authorized(self.cx.execute, "select * from test") + + class ProgressTests(MemoryDatabaseMixin, unittest.TestCase): + def assert_interrupted(self, func, /, *args, **kwargs): + with self.assertRaisesRegex(sqlite.OperationalError, "interrupted"): + func(*args, **kwargs) + def test_progress_handler_used(self): """ Test that the progress handler is invoked once it is set. @@ -219,11 +270,48 @@ def bad_progress(): create table foo(a, b) """) - def test_progress_handler_keyword_args(self): + def test_set_progress_handler_keyword_args(self): with self.assertRaisesRegex(TypeError, 'takes at least 1 positional argument'): self.con.set_progress_handler(progress_handler=lambda: None, n=1) + # When a handler has an invalid signature, the exception raised is + # the same that would be raised if the handler "negatively" replied. + + def test_progress_handler_invalid_signature(self): + self.cx.execute("create table if not exists test(a number)") + self.cx.set_progress_handler(lambda x: None, 1) + self.assert_interrupted(self.cx.execute, "select * from test") + + # Tests for checking that callback context mutations do not crash. + # Regression tests for https://github.com/python/cpython/issues/142830. + + @with_tracebacks(ZeroDivisionError, regex="hello world") + def test_progress_handler_concurrent_mutation_in_call(self): + self.cx.execute("create table if not exists test(a number)") + + def handler(*a, **kw): + self.cx.set_progress_handler(None, 1) + raise ZeroDivisionError("hello world") + + self.cx.set_progress_handler(handler, 1) + self.assert_interrupted(self.cx.execute, "select * from test") + + def test_progress_handler_concurrent_mutation_in_conversion(self): + self.cx.execute("create table if not exists test(a number)") + + class Handler: + def __bool__(_): + # clear the progress handler + self.cx.set_progress_handler(None, 1) + raise ValueError # force PyObject_True() to fail + + self.cx.set_progress_handler(Handler.__init__, 1) + self.assert_interrupted(self.cx.execute, "select * from test") + + # Running with tracebacks makes the second execution of this + # function raise another exception because of a database change. + class TraceCallbackTests(MemoryDatabaseMixin, unittest.TestCase): @@ -345,11 +433,40 @@ def test_trace_bad_handler(self): cx.set_trace_callback(lambda stmt: 5/0) cx.execute("select 1") - def test_trace_keyword_args(self): + def test_set_trace_callback_keyword_args(self): with self.assertRaisesRegex(TypeError, 'takes exactly 1 positional argument'): self.con.set_trace_callback(trace_callback=lambda: None) + # When a handler has an invalid signature, the exception raised is + # the same that would be raised if the handler "negatively" replied, + # but for the trace handler, exceptions are never re-raised (only + # printed when needed). + + @with_tracebacks( + TypeError, + regex=r".*\(\) missing 6 required positional arguments", + ) + def test_trace_handler_invalid_signature(self): + self.cx.execute("create table if not exists test(a number)") + self.cx.set_trace_callback(lambda x, y, z, t, a, b, c: None) + self.cx.execute("select * from test") + + # Tests for checking that callback context mutations do not crash. + # Regression tests for https://github.com/python/cpython/issues/142830. + + @with_tracebacks(ZeroDivisionError, regex="hello world") + def test_trace_callback_concurrent_mutation_in_call(self): + self.cx.execute("create table if not exists test(a number)") + + def handler(statement): + # clear the progress handler + self.cx.set_trace_callback(None) + raise ZeroDivisionError("hello world") + + self.cx.set_trace_callback(handler) + self.cx.execute("select * from test") + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2025-12-28-13-12-40.gh-issue-142830.uEyd6r.rst b/Misc/NEWS.d/next/Library/2025-12-28-13-12-40.gh-issue-142830.uEyd6r.rst new file mode 100644 index 00000000000000..246979e91d76b5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-28-13-12-40.gh-issue-142830.uEyd6r.rst @@ -0,0 +1,2 @@ +:mod:`sqlite3`: fix use-after-free crashes when the connection's callbacks +are mutated during a callback execution. Patch by Bénédikt Tran. diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 83ff8e60557c07..cde06c965ad4e3 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -145,7 +145,8 @@ class _sqlite3.Connection "pysqlite_Connection *" "clinic_state()->ConnectionTyp /*[clinic end generated code: output=da39a3ee5e6b4b0d input=67369db2faf80891]*/ static int _pysqlite_drop_unused_cursor_references(pysqlite_Connection* self); -static void free_callback_context(callback_context *ctx); +static void incref_callback_context(callback_context *ctx); +static void decref_callback_context(callback_context *ctx); static void set_callback_context(callback_context **ctx_pp, callback_context *ctx); static int connection_close(pysqlite_Connection *self); @@ -936,8 +937,9 @@ func_callback(sqlite3_context *context, int argc, sqlite3_value **argv) args = _pysqlite_build_py_params(context, argc, argv); if (args) { callback_context *ctx = (callback_context *)sqlite3_user_data(context); - assert(ctx != NULL); + incref_callback_context(ctx); py_retval = PyObject_CallObject(ctx->callable, args); + decref_callback_context(ctx); Py_DECREF(args); } @@ -964,7 +966,7 @@ step_callback(sqlite3_context *context, int argc, sqlite3_value **params) PyObject* stepmethod = NULL; callback_context *ctx = (callback_context *)sqlite3_user_data(context); - assert(ctx != NULL); + incref_callback_context(ctx); aggregate_instance = (PyObject**)sqlite3_aggregate_context(context, sizeof(PyObject*)); if (aggregate_instance == NULL) { @@ -1002,6 +1004,7 @@ step_callback(sqlite3_context *context, int argc, sqlite3_value **params) } error: + decref_callback_context(ctx); Py_XDECREF(stepmethod); Py_XDECREF(function_result); @@ -1033,9 +1036,10 @@ final_callback(sqlite3_context *context) PyObject *exc = PyErr_GetRaisedException(); callback_context *ctx = (callback_context *)sqlite3_user_data(context); - assert(ctx != NULL); + incref_callback_context(ctx); function_result = PyObject_CallMethodNoArgs(*aggregate_instance, ctx->state->str_finalize); + decref_callback_context(ctx); Py_DECREF(*aggregate_instance); ok = 0; @@ -1107,6 +1111,7 @@ create_callback_context(PyTypeObject *cls, PyObject *callable) callback_context *ctx = PyMem_Malloc(sizeof(callback_context)); if (ctx != NULL) { PyObject *module = PyType_GetModule(cls); + ctx->refcount = 1; ctx->callable = Py_NewRef(callable); ctx->module = Py_NewRef(module); ctx->state = pysqlite_get_state(module); @@ -1118,11 +1123,33 @@ static void free_callback_context(callback_context *ctx) { assert(ctx != NULL); + assert(ctx->refcount == 0); Py_XDECREF(ctx->callable); Py_XDECREF(ctx->module); PyMem_Free(ctx); } +static inline void +incref_callback_context(callback_context *ctx) +{ + assert(PyGILState_Check()); + assert(ctx != NULL); + assert(ctx->refcount > 0); + ctx->refcount++; +} + +static inline void +decref_callback_context(callback_context *ctx) +{ + assert(PyGILState_Check()); + assert(ctx != NULL); + assert(ctx->refcount > 0); + ctx->refcount--; + if (ctx->refcount == 0) { + free_callback_context(ctx); + } +} + static void set_callback_context(callback_context **ctx_pp, callback_context *ctx) { @@ -1130,7 +1157,7 @@ set_callback_context(callback_context **ctx_pp, callback_context *ctx) callback_context *tmp = *ctx_pp; *ctx_pp = ctx; if (tmp != NULL) { - free_callback_context(tmp); + decref_callback_context(tmp); } } @@ -1141,7 +1168,7 @@ destructor_callback(void *ctx) // This function may be called without the GIL held, so we need to // ensure that we destroy 'ctx' with the GIL held. PyGILState_STATE gstate = PyGILState_Ensure(); - free_callback_context((callback_context *)ctx); + decref_callback_context((callback_context *)ctx); PyGILState_Release(gstate); } } @@ -1202,7 +1229,7 @@ pysqlite_connection_create_function_impl(pysqlite_Connection *self, func_callback, NULL, NULL, - &destructor_callback); // will decref func + &destructor_callback); // will free 'ctx' if (rc != SQLITE_OK) { /* Workaround for SQLite bug: no error code or string is available here */ @@ -1226,7 +1253,7 @@ inverse_callback(sqlite3_context *context, int argc, sqlite3_value **params) PyGILState_STATE gilstate = PyGILState_Ensure(); callback_context *ctx = (callback_context *)sqlite3_user_data(context); - assert(ctx != NULL); + incref_callback_context(ctx); int size = sizeof(PyObject *); PyObject **cls = (PyObject **)sqlite3_aggregate_context(context, size); @@ -1258,6 +1285,7 @@ inverse_callback(sqlite3_context *context, int argc, sqlite3_value **params) Py_DECREF(res); exit: + decref_callback_context(ctx); Py_XDECREF(method); PyGILState_Release(gilstate); } @@ -1274,7 +1302,7 @@ value_callback(sqlite3_context *context) PyGILState_STATE gilstate = PyGILState_Ensure(); callback_context *ctx = (callback_context *)sqlite3_user_data(context); - assert(ctx != NULL); + incref_callback_context(ctx); int size = sizeof(PyObject *); PyObject **cls = (PyObject **)sqlite3_aggregate_context(context, size); @@ -1282,6 +1310,8 @@ value_callback(sqlite3_context *context) assert(*cls != NULL); PyObject *res = PyObject_CallMethodNoArgs(*cls, ctx->state->str_value); + decref_callback_context(ctx); + if (res == NULL) { int attr_err = PyErr_ExceptionMatches(PyExc_AttributeError); set_sqlite_error(context, attr_err @@ -1403,7 +1433,7 @@ pysqlite_connection_create_aggregate_impl(pysqlite_Connection *self, 0, &step_callback, &final_callback, - &destructor_callback); // will decref func + &destructor_callback); // will free 'ctx' if (rc != SQLITE_OK) { /* Workaround for SQLite bug: no error code or string is available here */ PyErr_SetString(self->OperationalError, "Error creating aggregate"); @@ -1413,7 +1443,7 @@ pysqlite_connection_create_aggregate_impl(pysqlite_Connection *self, } static int -authorizer_callback(void *ctx, int action, const char *arg1, +authorizer_callback(void *ctx_vp, int action, const char *arg1, const char *arg2 , const char *dbname, const char *access_attempt_source) { @@ -1422,8 +1452,9 @@ authorizer_callback(void *ctx, int action, const char *arg1, PyObject *ret; int rc = SQLITE_DENY; - assert(ctx != NULL); - PyObject *callable = ((callback_context *)ctx)->callable; + callback_context *ctx = (callback_context *)ctx_vp; + incref_callback_context(ctx); + PyObject *callable = ctx->callable; ret = PyObject_CallFunction(callable, "issss", action, arg1, arg2, dbname, access_attempt_source); @@ -1445,21 +1476,23 @@ authorizer_callback(void *ctx, int action, const char *arg1, Py_DECREF(ret); } + decref_callback_context(ctx); PyGILState_Release(gilstate); return rc; } static int -progress_callback(void *ctx) +progress_callback(void *ctx_vp) { PyGILState_STATE gilstate = PyGILState_Ensure(); int rc; PyObject *ret; - assert(ctx != NULL); - PyObject *callable = ((callback_context *)ctx)->callable; - ret = PyObject_CallNoArgs(callable); + callback_context *ctx = (callback_context *)ctx_vp; + incref_callback_context(ctx); + + ret = PyObject_CallNoArgs(ctx->callable); if (!ret) { /* abort query if error occurred */ rc = -1; @@ -1472,6 +1505,7 @@ progress_callback(void *ctx) print_or_clear_traceback(ctx); } + decref_callback_context(ctx); PyGILState_Release(gilstate); return rc; } @@ -1483,7 +1517,7 @@ progress_callback(void *ctx) * to ensure future compatibility. */ static int -trace_callback(unsigned int type, void *ctx, void *stmt, void *sql) +trace_callback(unsigned int type, void *ctx_vp, void *stmt, void *sql) { if (type != SQLITE_TRACE_STMT) { return 0; @@ -1491,8 +1525,9 @@ trace_callback(unsigned int type, void *ctx, void *stmt, void *sql) PyGILState_STATE gilstate = PyGILState_Ensure(); - assert(ctx != NULL); - pysqlite_state *state = ((callback_context *)ctx)->state; + callback_context *ctx = (callback_context *)ctx_vp; + incref_callback_context(ctx); + pysqlite_state *state = ctx->state; assert(state != NULL); PyObject *py_statement = NULL; @@ -1506,7 +1541,7 @@ trace_callback(unsigned int type, void *ctx, void *stmt, void *sql) PyErr_SetString(state->DataError, "Expanded SQL string exceeds the maximum string length"); - print_or_clear_traceback((callback_context *)ctx); + print_or_clear_traceback(ctx); // Fall back to unexpanded sql py_statement = PyUnicode_FromString((const char *)sql); @@ -1516,16 +1551,16 @@ trace_callback(unsigned int type, void *ctx, void *stmt, void *sql) sqlite3_free((void *)expanded_sql); } if (py_statement) { - PyObject *callable = ((callback_context *)ctx)->callable; - PyObject *ret = PyObject_CallOneArg(callable, py_statement); + PyObject *ret = PyObject_CallOneArg(ctx->callable, py_statement); Py_DECREF(py_statement); Py_XDECREF(ret); } if (PyErr_Occurred()) { - print_or_clear_traceback((callback_context *)ctx); + print_or_clear_traceback(ctx); } exit: + decref_callback_context(ctx); PyGILState_Release(gilstate); return 0; } @@ -1950,6 +1985,8 @@ collation_callback(void *context, int text1_length, const void *text1_data, PyObject* retval = NULL; long longval; int result = 0; + callback_context *ctx = (callback_context *)context; + incref_callback_context(ctx); /* This callback may be executed multiple times per sqlite3_step(). Bail if * the previous call failed */ @@ -1966,8 +2003,6 @@ collation_callback(void *context, int text1_length, const void *text1_data, goto finally; } - callback_context *ctx = (callback_context *)context; - assert(ctx != NULL); PyObject *args[] = { NULL, string1, string2 }; // Borrowed refs. size_t nargsf = 2 | PY_VECTORCALL_ARGUMENTS_OFFSET; retval = PyObject_Vectorcall(ctx->callable, args + 1, nargsf, NULL); @@ -1989,6 +2024,7 @@ collation_callback(void *context, int text1_length, const void *text1_data, } finally: + decref_callback_context(ctx); Py_XDECREF(string1); Py_XDECREF(string2); Py_XDECREF(retval); diff --git a/Modules/_sqlite/connection.h b/Modules/_sqlite/connection.h index 7a748ee3ea0c58..703396a0c8db53 100644 --- a/Modules/_sqlite/connection.h +++ b/Modules/_sqlite/connection.h @@ -36,6 +36,7 @@ typedef struct _callback_context PyObject *callable; PyObject *module; pysqlite_state *state; + Py_ssize_t refcount; } callback_context; enum autocommit_mode { From 2d9f4e357ad30e002ca0c7568047e9a818b96cd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartosz=20S=C5=82awecki?= Date: Thu, 1 Jan 2026 12:52:21 +0100 Subject: [PATCH 620/638] gh-143048: Remove outdated mention to `curses` in the "Interactive Mode" docs (#143049) --- Doc/tutorial/appendix.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Doc/tutorial/appendix.rst b/Doc/tutorial/appendix.rst index 5020c428741feb..2539edb51ba3a0 100644 --- a/Doc/tutorial/appendix.rst +++ b/Doc/tutorial/appendix.rst @@ -14,8 +14,7 @@ There are two variants of the interactive :term:`REPL`. The classic basic interpreter is supported on all platforms with minimal line control capabilities. -On Windows, or Unix-like systems with :mod:`curses` support, -a new interactive shell is used by default since Python 3.13. +Since Python 3.13, a new interactive shell is used by default. This one supports color, multiline editing, history browsing, and paste mode. To disable color, see :ref:`using-on-controlling-color` for details. Function keys provide some additional functionality. From 422ca074bc80a1d570d2599f733b8d8a748abf66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 1 Jan 2026 15:16:26 +0100 Subject: [PATCH 621/638] Amend NEWS entries for PRs GH-139553 and GH-142790 (#143329) --- .../2025-10-04-20-48-02.gh-issue-63016.EC9QN_.rst | 0 .../Library/2025-12-16-14-49-19.gh-issue-142783.VPV1ig.rst | 4 +++- 2 files changed, 3 insertions(+), 1 deletion(-) rename Misc/NEWS.d/next/{Core_and_Builtins => Library}/2025-10-04-20-48-02.gh-issue-63016.EC9QN_.rst (100%) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-04-20-48-02.gh-issue-63016.EC9QN_.rst b/Misc/NEWS.d/next/Library/2025-10-04-20-48-02.gh-issue-63016.EC9QN_.rst similarity index 100% rename from Misc/NEWS.d/next/Core_and_Builtins/2025-10-04-20-48-02.gh-issue-63016.EC9QN_.rst rename to Misc/NEWS.d/next/Library/2025-10-04-20-48-02.gh-issue-63016.EC9QN_.rst diff --git a/Misc/NEWS.d/next/Library/2025-12-16-14-49-19.gh-issue-142783.VPV1ig.rst b/Misc/NEWS.d/next/Library/2025-12-16-14-49-19.gh-issue-142783.VPV1ig.rst index f014771ae9a146..db6de6e801f8a4 100644 --- a/Misc/NEWS.d/next/Library/2025-12-16-14-49-19.gh-issue-142783.VPV1ig.rst +++ b/Misc/NEWS.d/next/Library/2025-12-16-14-49-19.gh-issue-142783.VPV1ig.rst @@ -1 +1,3 @@ -Fix zoneinfo use-after-free with descriptor _weak_cache. a descriptor as _weak_cache could cause crashes during object creation. The fix ensures proper reference counting for descriptor-provided objects. +:mod:`zoneinfo`: fix a use-after-free crash when instantiating +:class:`~zoneinfo.ZoneInfo` objects with an invalid ``_weak_cache`` +descriptor. From 1fb8e0eb51acca6062a7d57e23055dc28b6a34b7 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Fri, 2 Jan 2026 02:25:38 +0900 Subject: [PATCH 622/638] gh-134584: Eliminate redundant refcounting from _CALL{_BUILTIN_O, _METHOD_DESCRIPTOR_O} (GH-143330) Co-authored-by: Ken Jin --- Include/internal/pycore_opcode_metadata.h | 4 +- Include/internal/pycore_uop_ids.h | 2 +- Include/internal/pycore_uop_metadata.h | 8 +-- Lib/test/test_capi/test_opt.py | 24 +++++++- Python/bytecodes.c | 18 ++++-- Python/executor_cases.c.h | 45 ++++++--------- Python/generated_cases.c.h | 68 ++++++++++++++--------- Python/optimizer_bytecodes.c | 31 ++++++++++- Python/optimizer_cases.c.h | 53 +++++++++++++++--- 9 files changed, 177 insertions(+), 76 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 424ec337eb8afd..adce512d32941f 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1125,7 +1125,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_NON_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [CALL_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, @@ -1371,7 +1371,7 @@ _PyOpcode_macro_expansion[256] = { [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_NOARGS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_O, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 5, .uops = { { _CALL_METHOD_DESCRIPTOR_O, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_NON_PY_GENERAL] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE, OPARG_SIMPLE, 3 }, { _CALL_NON_PY_GENERAL, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_PY_EXACT_ARGS] = { .nuops = 8, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_PY_GENERAL] = { .nuops = 6, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index ebeec6387a741a..a7da996a99ae2d 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -422,7 +422,7 @@ extern "C" { #define _CALL_METHOD_DESCRIPTOR_FAST_r01 616 #define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r01 617 #define _CALL_METHOD_DESCRIPTOR_NOARGS_r01 618 -#define _CALL_METHOD_DESCRIPTOR_O_r01 619 +#define _CALL_METHOD_DESCRIPTOR_O_r03 619 #define _CALL_NON_PY_GENERAL_r01 620 #define _CALL_STR_1_r32 621 #define _CALL_TUPLE_1_r32 622 diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 1838dd3f0977b2..8c65565890b557 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -288,7 +288,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CALL_ISINSTANCE] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GUARD_CALLABLE_LIST_APPEND] = HAS_DEOPT_FLAG, [_CALL_LIST_APPEND] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, - [_CALL_METHOD_DESCRIPTOR_O] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_O] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -2648,7 +2648,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { [_CALL_METHOD_DESCRIPTOR_O] = { .best = { 0, 0, 0, 0 }, .entries = { - { 1, 0, _CALL_METHOD_DESCRIPTOR_O_r01 }, + { 3, 0, _CALL_METHOD_DESCRIPTOR_O_r03 }, { -1, -1, -1 }, { -1, -1, -1 }, { -1, -1, -1 }, @@ -3764,7 +3764,7 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_CALL_LIST_APPEND_r13] = _CALL_LIST_APPEND, [_CALL_LIST_APPEND_r23] = _CALL_LIST_APPEND, [_CALL_LIST_APPEND_r33] = _CALL_LIST_APPEND, - [_CALL_METHOD_DESCRIPTOR_O_r01] = _CALL_METHOD_DESCRIPTOR_O, + [_CALL_METHOD_DESCRIPTOR_O_r03] = _CALL_METHOD_DESCRIPTOR_O, [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r01] = _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, [_CALL_METHOD_DESCRIPTOR_NOARGS_r01] = _CALL_METHOD_DESCRIPTOR_NOARGS, [_CALL_METHOD_DESCRIPTOR_FAST_r01] = _CALL_METHOD_DESCRIPTOR_FAST, @@ -4045,7 +4045,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_CALL_METHOD_DESCRIPTOR_NOARGS] = "_CALL_METHOD_DESCRIPTOR_NOARGS", [_CALL_METHOD_DESCRIPTOR_NOARGS_r01] = "_CALL_METHOD_DESCRIPTOR_NOARGS_r01", [_CALL_METHOD_DESCRIPTOR_O] = "_CALL_METHOD_DESCRIPTOR_O", - [_CALL_METHOD_DESCRIPTOR_O_r01] = "_CALL_METHOD_DESCRIPTOR_O_r01", + [_CALL_METHOD_DESCRIPTOR_O_r03] = "_CALL_METHOD_DESCRIPTOR_O_r03", [_CALL_NON_PY_GENERAL] = "_CALL_NON_PY_GENERAL", [_CALL_NON_PY_GENERAL_r01] = "_CALL_NON_PY_GENERAL_r01", [_CALL_STR_1] = "_CALL_STR_1", diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 3780bdb28c8c44..93433b841740b4 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -2201,7 +2201,8 @@ def test_call_builtin_o(self): def testfunc(n): x = 0 for _ in range(n): - y = abs(1) + my_abs = abs + y = my_abs(1) x += y return x @@ -2210,7 +2211,26 @@ def testfunc(n): self.assertIsNotNone(ex) uops = get_opnames(ex) self.assertIn("_CALL_BUILTIN_O", uops) - self.assertIn("_POP_TOP", uops) + self.assertNotIn("_POP_TOP", uops) + self.assertIn("_POP_TOP_NOP", uops) + + def test_call_method_descriptor_o(self): + def testfunc(n): + x = 0 + for _ in range(n): + y = (1, 2, 3) + z = y.count(2) + x += z + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + pop_tops = [opname for opname in iter_opnames(ex) if opname == "_POP_TOP"] + self.assertIn("_CALL_METHOD_DESCRIPTOR_O", uops) + self.assertIn("_POP_TOP_NOP", uops) + self.assertLessEqual(len(pop_tops), 1) def test_get_len_with_const_tuple(self): def testfunc(n): diff --git a/Python/bytecodes.c b/Python/bytecodes.c index ea09c0645aa39c..0b74a03a4e56d4 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4170,7 +4170,7 @@ dummy_func( _CALL_BUILTIN_CLASS + _CHECK_PERIODIC_AT_END; - op(_CALL_BUILTIN_O, (callable, self_or_null, args[oparg] -- res, a, c)) { + op(_CALL_BUILTIN_O, (callable, self_or_null, args[oparg] -- res, c, s)) { /* Builtin METH_O functions */ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); @@ -4193,8 +4193,8 @@ dummy_func( if (res_o == NULL) { ERROR_NO_POP(); } - a = arg; c = callable; + s = args[0]; INPUTS_DEAD(); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -4362,7 +4362,7 @@ dummy_func( none = PyStackRef_None; } - op(_CALL_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg] -- res)) { + op(_CALL_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg] -- res, c, s, a)) { PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int total_args = oparg; @@ -4390,8 +4390,13 @@ dummy_func( PyStackRef_AsPyObjectBorrow(arg_stackref)); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - DECREF_INPUTS(); - ERROR_IF(res_o == NULL); + if (res_o == NULL) { + ERROR_NO_POP(); + } + c = callable; + s = arguments[0]; + a = arguments[1]; + INPUTS_DEAD(); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -4399,6 +4404,9 @@ dummy_func( unused/1 + unused/2 + _CALL_METHOD_DESCRIPTOR_O + + POP_TOP + + POP_TOP + + POP_TOP + _CHECK_PERIODIC_AT_END; op(_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (callable, self_or_null, args[oparg] -- res)) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 4e8fa34c0b2c0d..adac2914803d46 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -13568,8 +13568,8 @@ _PyStackRef self_or_null; _PyStackRef callable; _PyStackRef res; - _PyStackRef a; _PyStackRef c; + _PyStackRef s; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; @@ -13612,11 +13612,11 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - a = arg; c = callable; + s = args[0]; res = PyStackRef_FromPyObjectSteal(res_o); - _tos_cache2 = c; - _tos_cache1 = a; + _tos_cache2 = s; + _tos_cache1 = c; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2 - oparg; @@ -14302,13 +14302,16 @@ break; } - case _CALL_METHOD_DESCRIPTOR_O_r01: { + case _CALL_METHOD_DESCRIPTOR_O_r03: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; _PyStackRef res; + _PyStackRef c; + _PyStackRef s; + _PyStackRef a; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; @@ -14359,33 +14362,21 @@ stack_pointer = _PyFrame_GetStackPointer(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } + c = callable; + s = arguments[0]; + a = arguments[1]; res = PyStackRef_FromPyObjectSteal(res_o); - _tos_cache0 = res; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); + _tos_cache2 = a; + _tos_cache1 = s; + _tos_cache0 = c; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index e63852aee1134c..6e4ba9e9ece07b 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2327,8 +2327,8 @@ _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef res; - _PyStackRef a; _PyStackRef c; + _PyStackRef s; _PyStackRef value; /* Skip 1 cache entry */ /* Skip 2 cache entries */ @@ -2374,15 +2374,15 @@ if (res_o == NULL) { JUMP_TO_LABEL(error); } - a = arg; c = callable; + s = args[0]; res = PyStackRef_FromPyObjectSteal(res_o); } // _POP_TOP { - value = c; + value = s; stack_pointer[-2 - oparg] = res; - stack_pointer[-1 - oparg] = a; + stack_pointer[-1 - oparg] = c; stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -2391,7 +2391,7 @@ } // _POP_TOP { - value = a; + value = c; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -3613,6 +3613,10 @@ _PyStackRef self_or_null; _PyStackRef *args; _PyStackRef res; + _PyStackRef c; + _PyStackRef s; + _PyStackRef a; + _PyStackRef value; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _CALL_METHOD_DESCRIPTOR_O @@ -3666,34 +3670,46 @@ stack_pointer = _PyFrame_GetStackPointer(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { JUMP_TO_LABEL(error); } + c = callable; + s = arguments[0]; + a = arguments[1]; res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC_AT_END + // _POP_TOP { - stack_pointer[0] = res; - stack_pointer += 1; + value = a; + stack_pointer[-2 - oparg] = res; + stack_pointer[-1 - oparg] = c; + stack_pointer[-oparg] = s; + stack_pointer += 1 - oparg; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _POP_TOP + { + value = s; + stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _POP_TOP + { + value = c; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _CHECK_PERIODIC_AT_END + { _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 3e37cb81388ae0..aaa786ae5fd724 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -131,7 +131,7 @@ dummy_func(void) { } op(_PUSH_NULL, (-- res)) { - res = sym_new_null(ctx); + res = PyJitRef_Borrow(sym_new_null(ctx)); } op(_GUARD_TOS_INT, (value -- value)) { @@ -1064,6 +1064,35 @@ dummy_func(void) { none = sym_new_const(ctx, Py_None); } + op(_CALL_BUILTIN_O, (callable, self_or_null, args[oparg] -- res, c, s)) { + res = sym_new_not_null(ctx); + c = callable; + if (sym_is_not_null(self_or_null)) { + args--; + s = args[0]; + } + else if (sym_is_null(self_or_null)) { + s = args[0]; + } + else { + s = sym_new_unknown(ctx); + } + } + + op(_CALL_METHOD_DESCRIPTOR_O, (callable, self_or_null, args[oparg] -- res, c, s, a)) { + res = sym_new_not_null(ctx); + c = callable; + if (sym_is_not_null(self_or_null)) { + args--; + s = args[0]; + a = args[1]; + } + else { + s = sym_new_unknown(ctx); + a = sym_new_unknown(ctx); + } + } + op(_GUARD_IS_FALSE_POP, (flag -- )) { if (sym_is_const(ctx, flag)) { PyObject *value = sym_get_const(ctx, flag); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index b043a9493cbd7c..87c2d1a779b990 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -185,7 +185,7 @@ case _PUSH_NULL: { JitOptRef res; - res = sym_new_null(ctx); + res = PyJitRef_Borrow(sym_new_null(ctx)); CHECK_STACK_BOUNDS(1); stack_pointer[0] = res; stack_pointer += 1; @@ -2764,16 +2764,31 @@ } case _CALL_BUILTIN_O: { + JitOptRef *args; + JitOptRef self_or_null; + JitOptRef callable; JitOptRef res; - JitOptRef a; JitOptRef c; + JitOptRef s; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; res = sym_new_not_null(ctx); - a = sym_new_not_null(ctx); - c = sym_new_not_null(ctx); + c = callable; + if (sym_is_not_null(self_or_null)) { + args--; + s = args[0]; + } + else if (sym_is_null(self_or_null)) { + s = args[0]; + } + else { + s = sym_new_unknown(ctx); + } CHECK_STACK_BOUNDS(1 - oparg); stack_pointer[-2 - oparg] = res; - stack_pointer[-1 - oparg] = a; - stack_pointer[-oparg] = c; + stack_pointer[-1 - oparg] = c; + stack_pointer[-oparg] = s; stack_pointer += 1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; @@ -2912,11 +2927,33 @@ } case _CALL_METHOD_DESCRIPTOR_O: { + JitOptRef *args; + JitOptRef self_or_null; + JitOptRef callable; JitOptRef res; + JitOptRef c; + JitOptRef s; + JitOptRef a; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; res = sym_new_not_null(ctx); - CHECK_STACK_BOUNDS(-1 - oparg); + c = callable; + if (sym_is_not_null(self_or_null)) { + args--; + s = args[0]; + a = args[1]; + } + else { + s = sym_new_unknown(ctx); + a = sym_new_unknown(ctx); + } + CHECK_STACK_BOUNDS(2 - oparg); stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; + stack_pointer[-1 - oparg] = c; + stack_pointer[-oparg] = s; + stack_pointer[1 - oparg] = a; + stack_pointer += 2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } From d00d39f58e522a8968a741876a5dad5505f2c4df Mon Sep 17 00:00:00 2001 From: Nadeshiko Manju Date: Fri, 2 Jan 2026 01:27:02 +0800 Subject: [PATCH 623/638] gh-134584: Eliminate redundant refcounting from `_LOAD_ATTR_SLOT` (GH-143320) Signed-off-by: Manjusaka Co-authored-by: Ken Jin --- Include/internal/pycore_opcode_metadata.h | 2 +- Include/internal/pycore_uop_ids.h | 604 +++++++++--------- Include/internal/pycore_uop_metadata.h | 18 +- Lib/test/test_capi/test_opt.py | 19 + ...-01-01-17-01-24.gh-issue-134584.nis8LC.rst | 1 + Python/bytecodes.c | 6 +- Python/executor_cases.c.h | 99 ++- Python/generated_cases.c.h | 13 +- Python/optimizer_bytecodes.c | 3 +- Python/optimizer_cases.c.h | 8 + 10 files changed, 442 insertions(+), 331 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-01-01-17-01-24.gh-issue-134584.nis8LC.rst diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index adce512d32941f..830b49352e23a3 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1433,7 +1433,7 @@ _PyOpcode_macro_expansion[256] = { [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_NONDESCRIPTOR_NO_DICT, 4, 5 } } }, [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, OPARG_SIMPLE, 3 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, 4, 5 } } }, [LOAD_ATTR_PROPERTY] = { .nuops = 5, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_PROPERTY_FRAME, 4, 5 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 9 }, { _PUSH_FRAME, OPARG_SIMPLE, 9 } } }, - [LOAD_ATTR_SLOT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_SLOT, 1, 3 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, + [LOAD_ATTR_SLOT] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_SLOT, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, [LOAD_ATTR_WITH_HINT] = { .nuops = 4, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_WITH_HINT, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, [LOAD_BUILD_CLASS] = { .nuops = 1, .uops = { { _LOAD_BUILD_CLASS, OPARG_SIMPLE, 0 } } }, [LOAD_COMMON_CONSTANT] = { .nuops = 1, .uops = { { _LOAD_COMMON_CONSTANT, OPARG_SIMPLE, 0 } } }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index a7da996a99ae2d..e6e6c8266024aa 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -818,307 +818,309 @@ extern "C" { #define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11 1012 #define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11 1013 #define _LOAD_ATTR_PROPERTY_FRAME_r11 1014 -#define _LOAD_ATTR_SLOT_r11 1015 -#define _LOAD_ATTR_WITH_HINT_r12 1016 -#define _LOAD_BUILD_CLASS_r01 1017 -#define _LOAD_BYTECODE_r00 1018 -#define _LOAD_COMMON_CONSTANT_r01 1019 -#define _LOAD_COMMON_CONSTANT_r12 1020 -#define _LOAD_COMMON_CONSTANT_r23 1021 -#define _LOAD_CONST_r01 1022 -#define _LOAD_CONST_r12 1023 -#define _LOAD_CONST_r23 1024 -#define _LOAD_CONST_INLINE_r01 1025 -#define _LOAD_CONST_INLINE_r12 1026 -#define _LOAD_CONST_INLINE_r23 1027 -#define _LOAD_CONST_INLINE_BORROW_r01 1028 -#define _LOAD_CONST_INLINE_BORROW_r12 1029 -#define _LOAD_CONST_INLINE_BORROW_r23 1030 -#define _LOAD_CONST_UNDER_INLINE_r02 1031 -#define _LOAD_CONST_UNDER_INLINE_r12 1032 -#define _LOAD_CONST_UNDER_INLINE_r23 1033 -#define _LOAD_CONST_UNDER_INLINE_BORROW_r02 1034 -#define _LOAD_CONST_UNDER_INLINE_BORROW_r12 1035 -#define _LOAD_CONST_UNDER_INLINE_BORROW_r23 1036 -#define _LOAD_DEREF_r01 1037 -#define _LOAD_FAST_r01 1038 -#define _LOAD_FAST_r12 1039 -#define _LOAD_FAST_r23 1040 -#define _LOAD_FAST_0_r01 1041 -#define _LOAD_FAST_0_r12 1042 -#define _LOAD_FAST_0_r23 1043 -#define _LOAD_FAST_1_r01 1044 -#define _LOAD_FAST_1_r12 1045 -#define _LOAD_FAST_1_r23 1046 -#define _LOAD_FAST_2_r01 1047 -#define _LOAD_FAST_2_r12 1048 -#define _LOAD_FAST_2_r23 1049 -#define _LOAD_FAST_3_r01 1050 -#define _LOAD_FAST_3_r12 1051 -#define _LOAD_FAST_3_r23 1052 -#define _LOAD_FAST_4_r01 1053 -#define _LOAD_FAST_4_r12 1054 -#define _LOAD_FAST_4_r23 1055 -#define _LOAD_FAST_5_r01 1056 -#define _LOAD_FAST_5_r12 1057 -#define _LOAD_FAST_5_r23 1058 -#define _LOAD_FAST_6_r01 1059 -#define _LOAD_FAST_6_r12 1060 -#define _LOAD_FAST_6_r23 1061 -#define _LOAD_FAST_7_r01 1062 -#define _LOAD_FAST_7_r12 1063 -#define _LOAD_FAST_7_r23 1064 -#define _LOAD_FAST_AND_CLEAR_r01 1065 -#define _LOAD_FAST_AND_CLEAR_r12 1066 -#define _LOAD_FAST_AND_CLEAR_r23 1067 -#define _LOAD_FAST_BORROW_r01 1068 -#define _LOAD_FAST_BORROW_r12 1069 -#define _LOAD_FAST_BORROW_r23 1070 -#define _LOAD_FAST_BORROW_0_r01 1071 -#define _LOAD_FAST_BORROW_0_r12 1072 -#define _LOAD_FAST_BORROW_0_r23 1073 -#define _LOAD_FAST_BORROW_1_r01 1074 -#define _LOAD_FAST_BORROW_1_r12 1075 -#define _LOAD_FAST_BORROW_1_r23 1076 -#define _LOAD_FAST_BORROW_2_r01 1077 -#define _LOAD_FAST_BORROW_2_r12 1078 -#define _LOAD_FAST_BORROW_2_r23 1079 -#define _LOAD_FAST_BORROW_3_r01 1080 -#define _LOAD_FAST_BORROW_3_r12 1081 -#define _LOAD_FAST_BORROW_3_r23 1082 -#define _LOAD_FAST_BORROW_4_r01 1083 -#define _LOAD_FAST_BORROW_4_r12 1084 -#define _LOAD_FAST_BORROW_4_r23 1085 -#define _LOAD_FAST_BORROW_5_r01 1086 -#define _LOAD_FAST_BORROW_5_r12 1087 -#define _LOAD_FAST_BORROW_5_r23 1088 -#define _LOAD_FAST_BORROW_6_r01 1089 -#define _LOAD_FAST_BORROW_6_r12 1090 -#define _LOAD_FAST_BORROW_6_r23 1091 -#define _LOAD_FAST_BORROW_7_r01 1092 -#define _LOAD_FAST_BORROW_7_r12 1093 -#define _LOAD_FAST_BORROW_7_r23 1094 -#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r02 1095 -#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r13 1096 -#define _LOAD_FAST_CHECK_r01 1097 -#define _LOAD_FAST_CHECK_r12 1098 -#define _LOAD_FAST_CHECK_r23 1099 -#define _LOAD_FAST_LOAD_FAST_r02 1100 -#define _LOAD_FAST_LOAD_FAST_r13 1101 -#define _LOAD_FROM_DICT_OR_DEREF_r11 1102 -#define _LOAD_FROM_DICT_OR_GLOBALS_r11 1103 -#define _LOAD_GLOBAL_r00 1104 -#define _LOAD_GLOBAL_BUILTINS_r01 1105 -#define _LOAD_GLOBAL_MODULE_r01 1106 -#define _LOAD_LOCALS_r01 1107 -#define _LOAD_LOCALS_r12 1108 -#define _LOAD_LOCALS_r23 1109 -#define _LOAD_NAME_r01 1110 -#define _LOAD_SMALL_INT_r01 1111 -#define _LOAD_SMALL_INT_r12 1112 -#define _LOAD_SMALL_INT_r23 1113 -#define _LOAD_SMALL_INT_0_r01 1114 -#define _LOAD_SMALL_INT_0_r12 1115 -#define _LOAD_SMALL_INT_0_r23 1116 -#define _LOAD_SMALL_INT_1_r01 1117 -#define _LOAD_SMALL_INT_1_r12 1118 -#define _LOAD_SMALL_INT_1_r23 1119 -#define _LOAD_SMALL_INT_2_r01 1120 -#define _LOAD_SMALL_INT_2_r12 1121 -#define _LOAD_SMALL_INT_2_r23 1122 -#define _LOAD_SMALL_INT_3_r01 1123 -#define _LOAD_SMALL_INT_3_r12 1124 -#define _LOAD_SMALL_INT_3_r23 1125 -#define _LOAD_SPECIAL_r00 1126 -#define _LOAD_SUPER_ATTR_ATTR_r31 1127 -#define _LOAD_SUPER_ATTR_METHOD_r32 1128 -#define _MAKE_CALLARGS_A_TUPLE_r33 1129 -#define _MAKE_CELL_r00 1130 -#define _MAKE_FUNCTION_r11 1131 -#define _MAKE_WARM_r00 1132 -#define _MAKE_WARM_r11 1133 -#define _MAKE_WARM_r22 1134 -#define _MAKE_WARM_r33 1135 -#define _MAP_ADD_r20 1136 -#define _MATCH_CLASS_r31 1137 -#define _MATCH_KEYS_r23 1138 -#define _MATCH_MAPPING_r02 1139 -#define _MATCH_MAPPING_r12 1140 -#define _MATCH_MAPPING_r23 1141 -#define _MATCH_SEQUENCE_r02 1142 -#define _MATCH_SEQUENCE_r12 1143 -#define _MATCH_SEQUENCE_r23 1144 -#define _MAYBE_EXPAND_METHOD_r00 1145 -#define _MAYBE_EXPAND_METHOD_KW_r11 1146 -#define _MONITOR_CALL_r00 1147 -#define _MONITOR_CALL_KW_r11 1148 -#define _MONITOR_JUMP_BACKWARD_r00 1149 -#define _MONITOR_JUMP_BACKWARD_r11 1150 -#define _MONITOR_JUMP_BACKWARD_r22 1151 -#define _MONITOR_JUMP_BACKWARD_r33 1152 -#define _MONITOR_RESUME_r00 1153 -#define _NOP_r00 1154 -#define _NOP_r11 1155 -#define _NOP_r22 1156 -#define _NOP_r33 1157 -#define _POP_CALL_r20 1158 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW_r21 1159 -#define _POP_CALL_ONE_r30 1160 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW_r31 1161 -#define _POP_CALL_TWO_r30 1162 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW_r31 1163 -#define _POP_EXCEPT_r10 1164 -#define _POP_ITER_r20 1165 -#define _POP_JUMP_IF_FALSE_r00 1166 -#define _POP_JUMP_IF_FALSE_r10 1167 -#define _POP_JUMP_IF_FALSE_r21 1168 -#define _POP_JUMP_IF_FALSE_r32 1169 -#define _POP_JUMP_IF_TRUE_r00 1170 -#define _POP_JUMP_IF_TRUE_r10 1171 -#define _POP_JUMP_IF_TRUE_r21 1172 -#define _POP_JUMP_IF_TRUE_r32 1173 -#define _POP_TOP_r10 1174 -#define _POP_TOP_FLOAT_r00 1175 -#define _POP_TOP_FLOAT_r10 1176 -#define _POP_TOP_FLOAT_r21 1177 -#define _POP_TOP_FLOAT_r32 1178 -#define _POP_TOP_INT_r00 1179 -#define _POP_TOP_INT_r10 1180 -#define _POP_TOP_INT_r21 1181 -#define _POP_TOP_INT_r32 1182 -#define _POP_TOP_LOAD_CONST_INLINE_r11 1183 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW_r11 1184 -#define _POP_TOP_NOP_r00 1185 -#define _POP_TOP_NOP_r10 1186 -#define _POP_TOP_NOP_r21 1187 -#define _POP_TOP_NOP_r32 1188 -#define _POP_TOP_UNICODE_r00 1189 -#define _POP_TOP_UNICODE_r10 1190 -#define _POP_TOP_UNICODE_r21 1191 -#define _POP_TOP_UNICODE_r32 1192 -#define _POP_TWO_r20 1193 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW_r21 1194 -#define _PUSH_EXC_INFO_r02 1195 -#define _PUSH_EXC_INFO_r12 1196 -#define _PUSH_EXC_INFO_r23 1197 -#define _PUSH_FRAME_r10 1198 -#define _PUSH_NULL_r01 1199 -#define _PUSH_NULL_r12 1200 -#define _PUSH_NULL_r23 1201 -#define _PUSH_NULL_CONDITIONAL_r00 1202 -#define _PY_FRAME_GENERAL_r01 1203 -#define _PY_FRAME_KW_r11 1204 -#define _QUICKEN_RESUME_r00 1205 -#define _QUICKEN_RESUME_r11 1206 -#define _QUICKEN_RESUME_r22 1207 -#define _QUICKEN_RESUME_r33 1208 -#define _REPLACE_WITH_TRUE_r11 1209 -#define _RESUME_CHECK_r00 1210 -#define _RESUME_CHECK_r11 1211 -#define _RESUME_CHECK_r22 1212 -#define _RESUME_CHECK_r33 1213 -#define _RETURN_GENERATOR_r01 1214 -#define _RETURN_VALUE_r11 1215 -#define _SAVE_RETURN_OFFSET_r00 1216 -#define _SAVE_RETURN_OFFSET_r11 1217 -#define _SAVE_RETURN_OFFSET_r22 1218 -#define _SAVE_RETURN_OFFSET_r33 1219 -#define _SEND_r22 1220 -#define _SEND_GEN_FRAME_r22 1221 -#define _SETUP_ANNOTATIONS_r00 1222 -#define _SET_ADD_r10 1223 -#define _SET_FUNCTION_ATTRIBUTE_r01 1224 -#define _SET_FUNCTION_ATTRIBUTE_r11 1225 -#define _SET_FUNCTION_ATTRIBUTE_r21 1226 -#define _SET_FUNCTION_ATTRIBUTE_r32 1227 -#define _SET_IP_r00 1228 -#define _SET_IP_r11 1229 -#define _SET_IP_r22 1230 -#define _SET_IP_r33 1231 -#define _SET_UPDATE_r10 1232 -#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r02 1233 -#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r12 1234 -#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r22 1235 -#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r32 1236 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03 1237 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13 1238 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23 1239 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33 1240 -#define _SPILL_OR_RELOAD_r01 1241 -#define _SPILL_OR_RELOAD_r02 1242 -#define _SPILL_OR_RELOAD_r03 1243 -#define _SPILL_OR_RELOAD_r10 1244 -#define _SPILL_OR_RELOAD_r12 1245 -#define _SPILL_OR_RELOAD_r13 1246 -#define _SPILL_OR_RELOAD_r20 1247 -#define _SPILL_OR_RELOAD_r21 1248 -#define _SPILL_OR_RELOAD_r23 1249 -#define _SPILL_OR_RELOAD_r30 1250 -#define _SPILL_OR_RELOAD_r31 1251 -#define _SPILL_OR_RELOAD_r32 1252 -#define _START_EXECUTOR_r00 1253 -#define _STORE_ATTR_r20 1254 -#define _STORE_ATTR_INSTANCE_VALUE_r21 1255 -#define _STORE_ATTR_SLOT_r21 1256 -#define _STORE_ATTR_WITH_HINT_r21 1257 -#define _STORE_DEREF_r10 1258 -#define _STORE_FAST_r10 1259 -#define _STORE_FAST_0_r10 1260 -#define _STORE_FAST_1_r10 1261 -#define _STORE_FAST_2_r10 1262 -#define _STORE_FAST_3_r10 1263 -#define _STORE_FAST_4_r10 1264 -#define _STORE_FAST_5_r10 1265 -#define _STORE_FAST_6_r10 1266 -#define _STORE_FAST_7_r10 1267 -#define _STORE_FAST_LOAD_FAST_r11 1268 -#define _STORE_FAST_STORE_FAST_r20 1269 -#define _STORE_GLOBAL_r10 1270 -#define _STORE_NAME_r10 1271 -#define _STORE_SLICE_r30 1272 -#define _STORE_SUBSCR_r30 1273 -#define _STORE_SUBSCR_DICT_r31 1274 -#define _STORE_SUBSCR_LIST_INT_r32 1275 -#define _SWAP_r11 1276 -#define _SWAP_2_r02 1277 -#define _SWAP_2_r12 1278 -#define _SWAP_2_r22 1279 -#define _SWAP_2_r33 1280 -#define _SWAP_3_r03 1281 -#define _SWAP_3_r13 1282 -#define _SWAP_3_r23 1283 -#define _SWAP_3_r33 1284 -#define _TIER2_RESUME_CHECK_r00 1285 -#define _TIER2_RESUME_CHECK_r11 1286 -#define _TIER2_RESUME_CHECK_r22 1287 -#define _TIER2_RESUME_CHECK_r33 1288 -#define _TO_BOOL_r11 1289 -#define _TO_BOOL_BOOL_r01 1290 -#define _TO_BOOL_BOOL_r11 1291 -#define _TO_BOOL_BOOL_r22 1292 -#define _TO_BOOL_BOOL_r33 1293 -#define _TO_BOOL_INT_r11 1294 -#define _TO_BOOL_LIST_r11 1295 -#define _TO_BOOL_NONE_r01 1296 -#define _TO_BOOL_NONE_r11 1297 -#define _TO_BOOL_NONE_r22 1298 -#define _TO_BOOL_NONE_r33 1299 -#define _TO_BOOL_STR_r11 1300 -#define _TRACE_RECORD_r00 1301 -#define _UNARY_INVERT_r11 1302 -#define _UNARY_NEGATIVE_r11 1303 -#define _UNARY_NOT_r01 1304 -#define _UNARY_NOT_r11 1305 -#define _UNARY_NOT_r22 1306 -#define _UNARY_NOT_r33 1307 -#define _UNPACK_EX_r10 1308 -#define _UNPACK_SEQUENCE_r10 1309 -#define _UNPACK_SEQUENCE_LIST_r10 1310 -#define _UNPACK_SEQUENCE_TUPLE_r10 1311 -#define _UNPACK_SEQUENCE_TWO_TUPLE_r12 1312 -#define _WITH_EXCEPT_START_r33 1313 -#define _YIELD_VALUE_r11 1314 -#define MAX_UOP_REGS_ID 1314 +#define _LOAD_ATTR_SLOT_r02 1015 +#define _LOAD_ATTR_SLOT_r12 1016 +#define _LOAD_ATTR_SLOT_r23 1017 +#define _LOAD_ATTR_WITH_HINT_r12 1018 +#define _LOAD_BUILD_CLASS_r01 1019 +#define _LOAD_BYTECODE_r00 1020 +#define _LOAD_COMMON_CONSTANT_r01 1021 +#define _LOAD_COMMON_CONSTANT_r12 1022 +#define _LOAD_COMMON_CONSTANT_r23 1023 +#define _LOAD_CONST_r01 1024 +#define _LOAD_CONST_r12 1025 +#define _LOAD_CONST_r23 1026 +#define _LOAD_CONST_INLINE_r01 1027 +#define _LOAD_CONST_INLINE_r12 1028 +#define _LOAD_CONST_INLINE_r23 1029 +#define _LOAD_CONST_INLINE_BORROW_r01 1030 +#define _LOAD_CONST_INLINE_BORROW_r12 1031 +#define _LOAD_CONST_INLINE_BORROW_r23 1032 +#define _LOAD_CONST_UNDER_INLINE_r02 1033 +#define _LOAD_CONST_UNDER_INLINE_r12 1034 +#define _LOAD_CONST_UNDER_INLINE_r23 1035 +#define _LOAD_CONST_UNDER_INLINE_BORROW_r02 1036 +#define _LOAD_CONST_UNDER_INLINE_BORROW_r12 1037 +#define _LOAD_CONST_UNDER_INLINE_BORROW_r23 1038 +#define _LOAD_DEREF_r01 1039 +#define _LOAD_FAST_r01 1040 +#define _LOAD_FAST_r12 1041 +#define _LOAD_FAST_r23 1042 +#define _LOAD_FAST_0_r01 1043 +#define _LOAD_FAST_0_r12 1044 +#define _LOAD_FAST_0_r23 1045 +#define _LOAD_FAST_1_r01 1046 +#define _LOAD_FAST_1_r12 1047 +#define _LOAD_FAST_1_r23 1048 +#define _LOAD_FAST_2_r01 1049 +#define _LOAD_FAST_2_r12 1050 +#define _LOAD_FAST_2_r23 1051 +#define _LOAD_FAST_3_r01 1052 +#define _LOAD_FAST_3_r12 1053 +#define _LOAD_FAST_3_r23 1054 +#define _LOAD_FAST_4_r01 1055 +#define _LOAD_FAST_4_r12 1056 +#define _LOAD_FAST_4_r23 1057 +#define _LOAD_FAST_5_r01 1058 +#define _LOAD_FAST_5_r12 1059 +#define _LOAD_FAST_5_r23 1060 +#define _LOAD_FAST_6_r01 1061 +#define _LOAD_FAST_6_r12 1062 +#define _LOAD_FAST_6_r23 1063 +#define _LOAD_FAST_7_r01 1064 +#define _LOAD_FAST_7_r12 1065 +#define _LOAD_FAST_7_r23 1066 +#define _LOAD_FAST_AND_CLEAR_r01 1067 +#define _LOAD_FAST_AND_CLEAR_r12 1068 +#define _LOAD_FAST_AND_CLEAR_r23 1069 +#define _LOAD_FAST_BORROW_r01 1070 +#define _LOAD_FAST_BORROW_r12 1071 +#define _LOAD_FAST_BORROW_r23 1072 +#define _LOAD_FAST_BORROW_0_r01 1073 +#define _LOAD_FAST_BORROW_0_r12 1074 +#define _LOAD_FAST_BORROW_0_r23 1075 +#define _LOAD_FAST_BORROW_1_r01 1076 +#define _LOAD_FAST_BORROW_1_r12 1077 +#define _LOAD_FAST_BORROW_1_r23 1078 +#define _LOAD_FAST_BORROW_2_r01 1079 +#define _LOAD_FAST_BORROW_2_r12 1080 +#define _LOAD_FAST_BORROW_2_r23 1081 +#define _LOAD_FAST_BORROW_3_r01 1082 +#define _LOAD_FAST_BORROW_3_r12 1083 +#define _LOAD_FAST_BORROW_3_r23 1084 +#define _LOAD_FAST_BORROW_4_r01 1085 +#define _LOAD_FAST_BORROW_4_r12 1086 +#define _LOAD_FAST_BORROW_4_r23 1087 +#define _LOAD_FAST_BORROW_5_r01 1088 +#define _LOAD_FAST_BORROW_5_r12 1089 +#define _LOAD_FAST_BORROW_5_r23 1090 +#define _LOAD_FAST_BORROW_6_r01 1091 +#define _LOAD_FAST_BORROW_6_r12 1092 +#define _LOAD_FAST_BORROW_6_r23 1093 +#define _LOAD_FAST_BORROW_7_r01 1094 +#define _LOAD_FAST_BORROW_7_r12 1095 +#define _LOAD_FAST_BORROW_7_r23 1096 +#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r02 1097 +#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r13 1098 +#define _LOAD_FAST_CHECK_r01 1099 +#define _LOAD_FAST_CHECK_r12 1100 +#define _LOAD_FAST_CHECK_r23 1101 +#define _LOAD_FAST_LOAD_FAST_r02 1102 +#define _LOAD_FAST_LOAD_FAST_r13 1103 +#define _LOAD_FROM_DICT_OR_DEREF_r11 1104 +#define _LOAD_FROM_DICT_OR_GLOBALS_r11 1105 +#define _LOAD_GLOBAL_r00 1106 +#define _LOAD_GLOBAL_BUILTINS_r01 1107 +#define _LOAD_GLOBAL_MODULE_r01 1108 +#define _LOAD_LOCALS_r01 1109 +#define _LOAD_LOCALS_r12 1110 +#define _LOAD_LOCALS_r23 1111 +#define _LOAD_NAME_r01 1112 +#define _LOAD_SMALL_INT_r01 1113 +#define _LOAD_SMALL_INT_r12 1114 +#define _LOAD_SMALL_INT_r23 1115 +#define _LOAD_SMALL_INT_0_r01 1116 +#define _LOAD_SMALL_INT_0_r12 1117 +#define _LOAD_SMALL_INT_0_r23 1118 +#define _LOAD_SMALL_INT_1_r01 1119 +#define _LOAD_SMALL_INT_1_r12 1120 +#define _LOAD_SMALL_INT_1_r23 1121 +#define _LOAD_SMALL_INT_2_r01 1122 +#define _LOAD_SMALL_INT_2_r12 1123 +#define _LOAD_SMALL_INT_2_r23 1124 +#define _LOAD_SMALL_INT_3_r01 1125 +#define _LOAD_SMALL_INT_3_r12 1126 +#define _LOAD_SMALL_INT_3_r23 1127 +#define _LOAD_SPECIAL_r00 1128 +#define _LOAD_SUPER_ATTR_ATTR_r31 1129 +#define _LOAD_SUPER_ATTR_METHOD_r32 1130 +#define _MAKE_CALLARGS_A_TUPLE_r33 1131 +#define _MAKE_CELL_r00 1132 +#define _MAKE_FUNCTION_r11 1133 +#define _MAKE_WARM_r00 1134 +#define _MAKE_WARM_r11 1135 +#define _MAKE_WARM_r22 1136 +#define _MAKE_WARM_r33 1137 +#define _MAP_ADD_r20 1138 +#define _MATCH_CLASS_r31 1139 +#define _MATCH_KEYS_r23 1140 +#define _MATCH_MAPPING_r02 1141 +#define _MATCH_MAPPING_r12 1142 +#define _MATCH_MAPPING_r23 1143 +#define _MATCH_SEQUENCE_r02 1144 +#define _MATCH_SEQUENCE_r12 1145 +#define _MATCH_SEQUENCE_r23 1146 +#define _MAYBE_EXPAND_METHOD_r00 1147 +#define _MAYBE_EXPAND_METHOD_KW_r11 1148 +#define _MONITOR_CALL_r00 1149 +#define _MONITOR_CALL_KW_r11 1150 +#define _MONITOR_JUMP_BACKWARD_r00 1151 +#define _MONITOR_JUMP_BACKWARD_r11 1152 +#define _MONITOR_JUMP_BACKWARD_r22 1153 +#define _MONITOR_JUMP_BACKWARD_r33 1154 +#define _MONITOR_RESUME_r00 1155 +#define _NOP_r00 1156 +#define _NOP_r11 1157 +#define _NOP_r22 1158 +#define _NOP_r33 1159 +#define _POP_CALL_r20 1160 +#define _POP_CALL_LOAD_CONST_INLINE_BORROW_r21 1161 +#define _POP_CALL_ONE_r30 1162 +#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW_r31 1163 +#define _POP_CALL_TWO_r30 1164 +#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW_r31 1165 +#define _POP_EXCEPT_r10 1166 +#define _POP_ITER_r20 1167 +#define _POP_JUMP_IF_FALSE_r00 1168 +#define _POP_JUMP_IF_FALSE_r10 1169 +#define _POP_JUMP_IF_FALSE_r21 1170 +#define _POP_JUMP_IF_FALSE_r32 1171 +#define _POP_JUMP_IF_TRUE_r00 1172 +#define _POP_JUMP_IF_TRUE_r10 1173 +#define _POP_JUMP_IF_TRUE_r21 1174 +#define _POP_JUMP_IF_TRUE_r32 1175 +#define _POP_TOP_r10 1176 +#define _POP_TOP_FLOAT_r00 1177 +#define _POP_TOP_FLOAT_r10 1178 +#define _POP_TOP_FLOAT_r21 1179 +#define _POP_TOP_FLOAT_r32 1180 +#define _POP_TOP_INT_r00 1181 +#define _POP_TOP_INT_r10 1182 +#define _POP_TOP_INT_r21 1183 +#define _POP_TOP_INT_r32 1184 +#define _POP_TOP_LOAD_CONST_INLINE_r11 1185 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW_r11 1186 +#define _POP_TOP_NOP_r00 1187 +#define _POP_TOP_NOP_r10 1188 +#define _POP_TOP_NOP_r21 1189 +#define _POP_TOP_NOP_r32 1190 +#define _POP_TOP_UNICODE_r00 1191 +#define _POP_TOP_UNICODE_r10 1192 +#define _POP_TOP_UNICODE_r21 1193 +#define _POP_TOP_UNICODE_r32 1194 +#define _POP_TWO_r20 1195 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW_r21 1196 +#define _PUSH_EXC_INFO_r02 1197 +#define _PUSH_EXC_INFO_r12 1198 +#define _PUSH_EXC_INFO_r23 1199 +#define _PUSH_FRAME_r10 1200 +#define _PUSH_NULL_r01 1201 +#define _PUSH_NULL_r12 1202 +#define _PUSH_NULL_r23 1203 +#define _PUSH_NULL_CONDITIONAL_r00 1204 +#define _PY_FRAME_GENERAL_r01 1205 +#define _PY_FRAME_KW_r11 1206 +#define _QUICKEN_RESUME_r00 1207 +#define _QUICKEN_RESUME_r11 1208 +#define _QUICKEN_RESUME_r22 1209 +#define _QUICKEN_RESUME_r33 1210 +#define _REPLACE_WITH_TRUE_r11 1211 +#define _RESUME_CHECK_r00 1212 +#define _RESUME_CHECK_r11 1213 +#define _RESUME_CHECK_r22 1214 +#define _RESUME_CHECK_r33 1215 +#define _RETURN_GENERATOR_r01 1216 +#define _RETURN_VALUE_r11 1217 +#define _SAVE_RETURN_OFFSET_r00 1218 +#define _SAVE_RETURN_OFFSET_r11 1219 +#define _SAVE_RETURN_OFFSET_r22 1220 +#define _SAVE_RETURN_OFFSET_r33 1221 +#define _SEND_r22 1222 +#define _SEND_GEN_FRAME_r22 1223 +#define _SETUP_ANNOTATIONS_r00 1224 +#define _SET_ADD_r10 1225 +#define _SET_FUNCTION_ATTRIBUTE_r01 1226 +#define _SET_FUNCTION_ATTRIBUTE_r11 1227 +#define _SET_FUNCTION_ATTRIBUTE_r21 1228 +#define _SET_FUNCTION_ATTRIBUTE_r32 1229 +#define _SET_IP_r00 1230 +#define _SET_IP_r11 1231 +#define _SET_IP_r22 1232 +#define _SET_IP_r33 1233 +#define _SET_UPDATE_r10 1234 +#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r02 1235 +#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r12 1236 +#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r22 1237 +#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r32 1238 +#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03 1239 +#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13 1240 +#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23 1241 +#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33 1242 +#define _SPILL_OR_RELOAD_r01 1243 +#define _SPILL_OR_RELOAD_r02 1244 +#define _SPILL_OR_RELOAD_r03 1245 +#define _SPILL_OR_RELOAD_r10 1246 +#define _SPILL_OR_RELOAD_r12 1247 +#define _SPILL_OR_RELOAD_r13 1248 +#define _SPILL_OR_RELOAD_r20 1249 +#define _SPILL_OR_RELOAD_r21 1250 +#define _SPILL_OR_RELOAD_r23 1251 +#define _SPILL_OR_RELOAD_r30 1252 +#define _SPILL_OR_RELOAD_r31 1253 +#define _SPILL_OR_RELOAD_r32 1254 +#define _START_EXECUTOR_r00 1255 +#define _STORE_ATTR_r20 1256 +#define _STORE_ATTR_INSTANCE_VALUE_r21 1257 +#define _STORE_ATTR_SLOT_r21 1258 +#define _STORE_ATTR_WITH_HINT_r21 1259 +#define _STORE_DEREF_r10 1260 +#define _STORE_FAST_r10 1261 +#define _STORE_FAST_0_r10 1262 +#define _STORE_FAST_1_r10 1263 +#define _STORE_FAST_2_r10 1264 +#define _STORE_FAST_3_r10 1265 +#define _STORE_FAST_4_r10 1266 +#define _STORE_FAST_5_r10 1267 +#define _STORE_FAST_6_r10 1268 +#define _STORE_FAST_7_r10 1269 +#define _STORE_FAST_LOAD_FAST_r11 1270 +#define _STORE_FAST_STORE_FAST_r20 1271 +#define _STORE_GLOBAL_r10 1272 +#define _STORE_NAME_r10 1273 +#define _STORE_SLICE_r30 1274 +#define _STORE_SUBSCR_r30 1275 +#define _STORE_SUBSCR_DICT_r31 1276 +#define _STORE_SUBSCR_LIST_INT_r32 1277 +#define _SWAP_r11 1278 +#define _SWAP_2_r02 1279 +#define _SWAP_2_r12 1280 +#define _SWAP_2_r22 1281 +#define _SWAP_2_r33 1282 +#define _SWAP_3_r03 1283 +#define _SWAP_3_r13 1284 +#define _SWAP_3_r23 1285 +#define _SWAP_3_r33 1286 +#define _TIER2_RESUME_CHECK_r00 1287 +#define _TIER2_RESUME_CHECK_r11 1288 +#define _TIER2_RESUME_CHECK_r22 1289 +#define _TIER2_RESUME_CHECK_r33 1290 +#define _TO_BOOL_r11 1291 +#define _TO_BOOL_BOOL_r01 1292 +#define _TO_BOOL_BOOL_r11 1293 +#define _TO_BOOL_BOOL_r22 1294 +#define _TO_BOOL_BOOL_r33 1295 +#define _TO_BOOL_INT_r11 1296 +#define _TO_BOOL_LIST_r11 1297 +#define _TO_BOOL_NONE_r01 1298 +#define _TO_BOOL_NONE_r11 1299 +#define _TO_BOOL_NONE_r22 1300 +#define _TO_BOOL_NONE_r33 1301 +#define _TO_BOOL_STR_r11 1302 +#define _TRACE_RECORD_r00 1303 +#define _UNARY_INVERT_r11 1304 +#define _UNARY_NEGATIVE_r11 1305 +#define _UNARY_NOT_r01 1306 +#define _UNARY_NOT_r11 1307 +#define _UNARY_NOT_r22 1308 +#define _UNARY_NOT_r33 1309 +#define _UNPACK_EX_r10 1310 +#define _UNPACK_SEQUENCE_r10 1311 +#define _UNPACK_SEQUENCE_LIST_r10 1312 +#define _UNPACK_SEQUENCE_TUPLE_r10 1313 +#define _UNPACK_SEQUENCE_TWO_TUPLE_r12 1314 +#define _WITH_EXCEPT_START_r33 1315 +#define _YIELD_VALUE_r11 1316 +#define MAX_UOP_REGS_ID 1316 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 8c65565890b557..214f58b22338e1 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -193,7 +193,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_ATTR_INSTANCE_VALUE] = HAS_DEOPT_FLAG, [_LOAD_ATTR_MODULE] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_LOAD_ATTR_WITH_HINT] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG, - [_LOAD_ATTR_SLOT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_ATTR_SLOT] = HAS_DEOPT_FLAG, [_CHECK_ATTR_CLASS] = HAS_EXIT_FLAG, [_LOAD_ATTR_CLASS] = HAS_ESCAPES_FLAG, [_LOAD_ATTR_PROPERTY_FRAME] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, @@ -1791,11 +1791,11 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { }, }, [_LOAD_ATTR_SLOT] = { - .best = { 1, 1, 1, 1 }, + .best = { 0, 1, 2, 2 }, .entries = { - { -1, -1, -1 }, - { 1, 1, _LOAD_ATTR_SLOT_r11 }, - { -1, -1, -1 }, + { 2, 0, _LOAD_ATTR_SLOT_r02 }, + { 2, 1, _LOAD_ATTR_SLOT_r12 }, + { 3, 2, _LOAD_ATTR_SLOT_r23 }, { -1, -1, -1 }, }, }, @@ -3569,7 +3569,9 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_LOAD_ATTR_INSTANCE_VALUE_r23] = _LOAD_ATTR_INSTANCE_VALUE, [_LOAD_ATTR_MODULE_r11] = _LOAD_ATTR_MODULE, [_LOAD_ATTR_WITH_HINT_r12] = _LOAD_ATTR_WITH_HINT, - [_LOAD_ATTR_SLOT_r11] = _LOAD_ATTR_SLOT, + [_LOAD_ATTR_SLOT_r02] = _LOAD_ATTR_SLOT, + [_LOAD_ATTR_SLOT_r12] = _LOAD_ATTR_SLOT, + [_LOAD_ATTR_SLOT_r23] = _LOAD_ATTR_SLOT, [_CHECK_ATTR_CLASS_r01] = _CHECK_ATTR_CLASS, [_CHECK_ATTR_CLASS_r11] = _CHECK_ATTR_CLASS, [_CHECK_ATTR_CLASS_r22] = _CHECK_ATTR_CLASS, @@ -4544,7 +4546,9 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_LOAD_ATTR_PROPERTY_FRAME] = "_LOAD_ATTR_PROPERTY_FRAME", [_LOAD_ATTR_PROPERTY_FRAME_r11] = "_LOAD_ATTR_PROPERTY_FRAME_r11", [_LOAD_ATTR_SLOT] = "_LOAD_ATTR_SLOT", - [_LOAD_ATTR_SLOT_r11] = "_LOAD_ATTR_SLOT_r11", + [_LOAD_ATTR_SLOT_r02] = "_LOAD_ATTR_SLOT_r02", + [_LOAD_ATTR_SLOT_r12] = "_LOAD_ATTR_SLOT_r12", + [_LOAD_ATTR_SLOT_r23] = "_LOAD_ATTR_SLOT_r23", [_LOAD_ATTR_WITH_HINT] = "_LOAD_ATTR_WITH_HINT", [_LOAD_ATTR_WITH_HINT_r12] = "_LOAD_ATTR_WITH_HINT_r12", [_LOAD_BUILD_CLASS] = "_LOAD_BUILD_CLASS", diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 93433b841740b4..bb4aa6ff113f47 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -2593,6 +2593,25 @@ class C: self.assertNotIn("_POP_TOP", uops) self.assertIn("_POP_TOP_NOP", uops) + def test_load_addr_slot(self): + def testfunc(n): + class C: + __slots__ = ('x',) + c = C() + c.x = 42 + x = 0 + for _ in range(n): + x += c.x + return x + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, 42 * TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + + self.assertIn("_LOAD_ATTR_SLOT", uops) + self.assertNotIn("_POP_TOP", uops) + self.assertIn("_POP_TOP_NOP", uops) + def test_int_add_op_refcount_elimination(self): def testfunc(n): c = 1 diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-01-01-17-01-24.gh-issue-134584.nis8LC.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-01-01-17-01-24.gh-issue-134584.nis8LC.rst new file mode 100644 index 00000000000000..ac5a2b6e8adaff --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-01-01-17-01-24.gh-issue-134584.nis8LC.rst @@ -0,0 +1 @@ +Eliminate redundant refcounting from ``_LOAD_ATTR_SLOT``. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 0b74a03a4e56d4..829efafa67d6e1 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2485,7 +2485,7 @@ dummy_func( unused/5 + _PUSH_NULL_CONDITIONAL; - op(_LOAD_ATTR_SLOT, (index/1, owner -- attr)) { + op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, o)) { PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **addr = (PyObject **)((char *)owner_o + index); @@ -2498,13 +2498,15 @@ dummy_func( attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); - DECREF_INPUTS(); + o = owner; + DEAD(owner); } macro(LOAD_ATTR_SLOT) = unused/1 + _GUARD_TYPE_VERSION + _LOAD_ATTR_SLOT + // NOTE: This action may also deopt + POP_TOP + unused/5 + _PUSH_NULL_CONDITIONAL; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index adac2914803d46..0a2b794988c961 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -8532,11 +8532,49 @@ break; } - case _LOAD_ATTR_SLOT_r11: { + case _LOAD_ATTR_SLOT_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef o; + owner = stack_pointer[-1]; + uint16_t index = (uint16_t)CURRENT_OPERAND0_16(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject **addr = (PyObject **)((char *)owner_o + index); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); + if (attr_o == NULL) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); + if (!increfed) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + o = owner; + _tos_cache1 = o; + _tos_cache0 = attr; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_ATTR_SLOT_r12: { CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef owner; _PyStackRef attr; + _PyStackRef o; _PyStackRef _stack_item_0 = _tos_cache0; owner = _stack_item_0; uint16_t index = (uint16_t)CURRENT_OPERAND0_16(); @@ -8561,21 +8599,52 @@ attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); - stack_pointer[0] = owner; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = owner; - owner = attr; - stack_pointer[-1] = owner; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + o = owner; + _tos_cache1 = o; _tos_cache0 = attr; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _LOAD_ATTR_SLOT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef o; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + owner = _stack_item_1; + uint16_t index = (uint16_t)CURRENT_OPERAND0_16(); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject **addr = (PyObject **)((char *)owner_o + index); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); + if (attr_o == NULL) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); + if (!increfed) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = owner; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + o = owner; + _tos_cache2 = o; + _tos_cache1 = attr; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 6e4ba9e9ece07b..716d87fd97ac4a 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -8284,6 +8284,8 @@ static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; + _PyStackRef o; + _PyStackRef value; _PyStackRef *null; /* Skip 1 cache entry */ // _GUARD_TYPE_VERSION @@ -8320,11 +8322,14 @@ attr = PyStackRef_FromPyObjectNew(attr_o); #endif STAT_INC(LOAD_ATTR, hit); + o = owner; + } + // _POP_TOP + { + value = o; + stack_pointer[-1] = attr; _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = owner; - owner = attr; - stack_pointer[-1] = owner; - PyStackRef_CLOSE(tmp); + PyStackRef_XCLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); } /* Skip 5 cache entries */ diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index aaa786ae5fd724..53c7cb724e1b65 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -661,9 +661,10 @@ dummy_func(void) { o = owner; } - op(_LOAD_ATTR_SLOT, (index/1, owner -- attr)) { + op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, o)) { attr = sym_new_not_null(ctx); (void)index; + o = owner; } op(_LOAD_ATTR_CLASS, (descr/4, owner -- attr)) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 87c2d1a779b990..49e6ac560306fe 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1666,11 +1666,19 @@ } case _LOAD_ATTR_SLOT: { + JitOptRef owner; JitOptRef attr; + JitOptRef o; + owner = stack_pointer[-1]; uint16_t index = (uint16_t)this_instr->operand0; attr = sym_new_not_null(ctx); (void)index; + o = owner; + CHECK_STACK_BOUNDS(1); stack_pointer[-1] = attr; + stack_pointer[0] = o; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } From faa26044ce8e61f15a7886557d3030a481ebb96d Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Fri, 2 Jan 2026 02:54:49 +0800 Subject: [PATCH 624/638] gh-134584: Fix _CALL_BUILTIN_O test to reflect real-world usage (GH-143333) Fix test to reflect real-world usage --- Lib/test/test_capi/test_opt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index bb4aa6ff113f47..bff97fe8320b22 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -2201,8 +2201,7 @@ def test_call_builtin_o(self): def testfunc(n): x = 0 for _ in range(n): - my_abs = abs - y = my_abs(1) + y = abs(1) x += y return x @@ -2210,9 +2209,10 @@ def testfunc(n): self.assertEqual(res, TIER2_THRESHOLD) self.assertIsNotNone(ex) uops = get_opnames(ex) + pop_tops = [opname for opname in iter_opnames(ex) if opname == "_POP_TOP"] self.assertIn("_CALL_BUILTIN_O", uops) - self.assertNotIn("_POP_TOP", uops) self.assertIn("_POP_TOP_NOP", uops) + self.assertLessEqual(len(pop_tops), 1) def test_call_method_descriptor_o(self): def testfunc(n): From 5d133351c63b20882d85f92c2942c7d99066cebb Mon Sep 17 00:00:00 2001 From: ivonastojanovic <80911834+ivonastojanovic@users.noreply.github.com> Date: Thu, 1 Jan 2026 20:05:45 +0100 Subject: [PATCH 625/638] gh-142927: Auto-open HTML output in browser after generation (#143178) --- Doc/library/profiling.sampling.rst | 7 ++++ Lib/profiling/sampling/cli.py | 41 +++++++++++++++++++ .../test_sampling_profiler/test_children.py | 5 +++ 3 files changed, 53 insertions(+) diff --git a/Doc/library/profiling.sampling.rst b/Doc/library/profiling.sampling.rst index dae67cca66d9b4..9bc58b4d1bc976 100644 --- a/Doc/library/profiling.sampling.rst +++ b/Doc/library/profiling.sampling.rst @@ -1490,6 +1490,13 @@ Output options named ``_.`` (for example, ``flamegraph_12345.html``). :option:`--heatmap` creates a directory named ``heatmap_``. +.. option:: --browser + + Automatically open HTML output (:option:`--flamegraph` and + :option:`--heatmap`) in your default web browser after generation. + When profiling with :option:`--subprocesses`, only the main process + opens the browser; subprocess outputs are never auto-opened. + pstats display options ---------------------- diff --git a/Lib/profiling/sampling/cli.py b/Lib/profiling/sampling/cli.py index e43925ea8595f0..ea3926c9565809 100644 --- a/Lib/profiling/sampling/cli.py +++ b/Lib/profiling/sampling/cli.py @@ -10,6 +10,7 @@ import subprocess import sys import time +import webbrowser from contextlib import nullcontext from .errors import SamplingUnknownProcessError, SamplingModuleNotFoundError, SamplingScriptNotFoundError @@ -487,6 +488,12 @@ def _add_format_options(parser, include_compression=True, include_binary=True): help="Output path (default: stdout for pstats, auto-generated for others). " "For heatmap: directory name (default: heatmap_PID)", ) + output_group.add_argument( + "--browser", + action="store_true", + help="Automatically open HTML output (flamegraph, heatmap) in browser. " + "When using --subprocesses, only the main process opens the browser", + ) def _add_pstats_options(parser): @@ -586,6 +593,32 @@ def _generate_output_filename(format_type, pid): return f"{format_type}_{pid}.{extension}" +def _open_in_browser(path): + """Open a file or directory in the default web browser. + + Args: + path: File path or directory path to open + + For directories (heatmap), opens the index.html file inside. + """ + abs_path = os.path.abspath(path) + + # For heatmap directories, open the index.html file + if os.path.isdir(abs_path): + index_path = os.path.join(abs_path, 'index.html') + if os.path.exists(index_path): + abs_path = index_path + else: + print(f"Warning: Could not find index.html in {path}", file=sys.stderr) + return + + file_url = f"file://{abs_path}" + try: + webbrowser.open(file_url) + except Exception as e: + print(f"Warning: Could not open browser: {e}", file=sys.stderr) + + def _handle_output(collector, args, pid, mode): """Handle output for the collector based on format and arguments. @@ -625,6 +658,10 @@ def _handle_output(collector, args, pid, mode): filename = args.outfile or _generate_output_filename(args.format, pid) collector.export(filename) + # Auto-open browser for HTML output if --browser flag is set + if args.format in ('flamegraph', 'heatmap') and getattr(args, 'browser', False): + _open_in_browser(filename) + def _validate_args(args, parser): """Validate format-specific options and live mode requirements. @@ -1161,6 +1198,10 @@ def progress_callback(current, total): filename = args.outfile or _generate_output_filename(args.format, os.getpid()) collector.export(filename) + # Auto-open browser for HTML output if --browser flag is set + if args.format in ('flamegraph', 'heatmap') and getattr(args, 'browser', False): + _open_in_browser(filename) + print(f"Replayed {count} samples") diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_children.py b/Lib/test/test_profiling/test_sampling_profiler/test_children.py index b7dc878a238f8d..84d50cd2088a9e 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_children.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_children.py @@ -438,6 +438,11 @@ def assert_flag_value_pair(flag, value): child_args, f"Flag '--flamegraph' not found in args: {child_args}", ) + self.assertNotIn( + "--browser", + child_args, + f"Flag '--browser' should not be in child args: {child_args}", + ) def test_build_child_profiler_args_no_gc(self): """Test building CLI args with --no-gc.""" From 513ae175bb4839f121b6e6806ec172437f3dcea1 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 1 Jan 2026 19:05:59 +0000 Subject: [PATCH 626/638] gh-142927: Fix heatmap caller navigation for interior lines (#143180) --- Lib/profiling/sampling/heatmap_collector.py | 37 ++++++++- Lib/test/test_profiling/test_heatmap.py | 90 +++++++++++++++++++++ 2 files changed, 124 insertions(+), 3 deletions(-) diff --git a/Lib/profiling/sampling/heatmap_collector.py b/Lib/profiling/sampling/heatmap_collector.py index bb810fa485be63..022e94d014f9b7 100644 --- a/Lib/profiling/sampling/heatmap_collector.py +++ b/Lib/profiling/sampling/heatmap_collector.py @@ -472,6 +472,10 @@ def __init__(self, *args, **kwargs): self.callers_graph = collections.defaultdict(set) self.function_definitions = {} + # Map each sampled line to its function for proper caller lookup + # (filename, lineno) -> funcname + self.line_to_function = {} + # Edge counting for call path analysis self.edge_samples = collections.Counter() @@ -596,6 +600,10 @@ def _record_line_sample(self, filename, lineno, funcname, is_leaf=False, if funcname and (filename, funcname) not in self.function_definitions: self.function_definitions[(filename, funcname)] = lineno + # Map this line to its function for caller/callee navigation + if funcname: + self.line_to_function[(filename, lineno)] = funcname + def _record_bytecode_sample(self, filename, lineno, opcode, end_lineno=None, col_offset=None, end_col_offset=None, weight=1): @@ -1150,13 +1158,36 @@ def _format_specialization_color(self, spec_pct: int) -> str: return f"rgba({r}, {g}, {b}, {alpha})" def _build_navigation_buttons(self, filename: str, line_num: int) -> str: - """Build navigation buttons for callers/callees.""" + """Build navigation buttons for callers/callees. + + - Callers: All lines in a function show who calls this function + - Callees: Only actual call site lines show what they call + """ line_key = (filename, line_num) - caller_list = self._deduplicate_by_function(self.callers_graph.get(line_key, set())) + + funcname = self.line_to_function.get(line_key) + + # Get callers: look up by function definition line, not current line + # This ensures all lines in a function show who calls this function + if funcname: + func_def_line = self.function_definitions.get((filename, funcname), line_num) + func_def_key = (filename, func_def_line) + caller_list = self._deduplicate_by_function(self.callers_graph.get(func_def_key, set())) + else: + caller_list = self._deduplicate_by_function(self.callers_graph.get(line_key, set())) + + # Get callees: only show for actual call site lines (not every line in function) callee_list = self._deduplicate_by_function(self.call_graph.get(line_key, set())) # Get edge counts for each caller/callee - callers_with_counts = self._get_edge_counts(line_key, caller_list, is_caller=True) + # For callers, use the function definition key for edge lookup + if funcname: + func_def_line = self.function_definitions.get((filename, funcname), line_num) + caller_edge_key = (filename, func_def_line) + else: + caller_edge_key = line_key + callers_with_counts = self._get_edge_counts(caller_edge_key, caller_list, is_caller=True) + # For callees, use the actual line key since that's where the call happens callees_with_counts = self._get_edge_counts(line_key, callee_list, is_caller=False) # Build navigation buttons with counts diff --git a/Lib/test/test_profiling/test_heatmap.py b/Lib/test/test_profiling/test_heatmap.py index b1bfdf868b085a..b2acb1cf577341 100644 --- a/Lib/test/test_profiling/test_heatmap.py +++ b/Lib/test/test_profiling/test_heatmap.py @@ -367,6 +367,96 @@ def test_process_frames_with_file_samples_dict(self): self.assertEqual(collector.file_samples['test.py'][10], 1) +def frame(filename, line, func): + """Create a frame tuple: (filename, location, funcname, opcode).""" + return (filename, (line, line, -1, -1), func, None) + + +class TestHeatmapCollectorNavigationButtons(unittest.TestCase): + """Test navigation button behavior for caller/callee relationships. + + For every call stack: + - Root frames (entry points): only DOWN arrow (callees) + - Middle frames: both UP and DOWN arrows + - Leaf frames: only UP arrow (callers) + """ + + def collect(self, *stacks): + """Create collector and process frame stacks.""" + collector = HeatmapCollector(sample_interval_usec=100) + for stack in stacks: + collector.process_frames(stack, thread_id=1) + return collector + + def test_deep_call_stack_relationships(self): + """Test root/middle/leaf navigation in a 5-level call stack.""" + # Stack: root -> A -> B -> C -> leaf + stack = [ + frame('leaf.py', 5, 'leaf'), + frame('c.py', 10, 'func_c'), + frame('b.py', 15, 'func_b'), + frame('a.py', 20, 'func_a'), + frame('root.py', 25, 'root'), + ] + c = self.collect(stack) + + # Root: only callees (no one calls it) + self.assertIn(('root.py', 25), c.call_graph) + self.assertNotIn(('root.py', 25), c.callers_graph) + + # Middle frames: both callers and callees + for key in [('a.py', 20), ('b.py', 15), ('c.py', 10)]: + self.assertIn(key, c.call_graph) + self.assertIn(key, c.callers_graph) + + # Leaf: only callers (doesn't call anyone) + self.assertNotIn(('leaf.py', 5), c.call_graph) + self.assertIn(('leaf.py', 5), c.callers_graph) + + def test_all_lines_in_function_see_callers(self): + """Test that interior lines map to their function for caller lookup.""" + # Same function sampled at different lines (12, 15, 10) + c = self.collect( + [frame('mod.py', 12, 'my_func'), frame('caller.py', 100, 'caller')], + [frame('mod.py', 15, 'my_func'), frame('caller.py', 100, 'caller')], + [frame('mod.py', 10, 'my_func'), frame('caller.py', 100, 'caller')], + ) + + # All lines should map to same function + for line in [10, 12, 15]: + self.assertEqual(c.line_to_function[('mod.py', line)], 'my_func') + + # Function definition line should have callers + func_def = c.function_definitions[('mod.py', 'my_func')] + self.assertIn(('mod.py', func_def), c.callers_graph) + + def test_multiple_callers_and_callees(self): + """Test multiple callers/callees are recorded correctly.""" + # Two callers -> target, and caller -> two callees + c = self.collect( + [frame('target.py', 10, 'target'), frame('caller1.py', 20, 'c1')], + [frame('target.py', 10, 'target'), frame('caller2.py', 30, 'c2')], + [frame('callee1.py', 5, 'f1'), frame('dispatcher.py', 40, 'dispatch')], + [frame('callee2.py', 6, 'f2'), frame('dispatcher.py', 40, 'dispatch')], + ) + + # Target has 2 callers + callers = c.callers_graph[('target.py', 10)] + self.assertEqual({x[0] for x in callers}, {'caller1.py', 'caller2.py'}) + + # Dispatcher has 2 callees + callees = c.call_graph[('dispatcher.py', 40)] + self.assertEqual({x[0] for x in callees}, {'callee1.py', 'callee2.py'}) + + def test_edge_samples_counted(self): + """Test that repeated calls accumulate edge counts.""" + stack = [frame('callee.py', 10, 'callee'), frame('caller.py', 20, 'caller')] + c = self.collect(stack, stack, stack) + + edge_key = (('caller.py', 20), ('callee.py', 10)) + self.assertEqual(c.edge_samples[edge_key], 3) + + class TestHeatmapCollectorExport(unittest.TestCase): """Test HeatmapCollector.export() method.""" From e5ad7b7694c47555e3eac3fcb227a4b1b7b781c4 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 1 Jan 2026 21:10:52 +0000 Subject: [PATCH 627/638] gh-138122: Integrate live profiler TUI with _colorize theming system (#142360) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Lib/_colorize.py | 130 +++++++++++++++++- .../sampling/live_collector/collector.py | 100 ++++++-------- .../sampling/live_collector/constants.py | 3 + .../sampling/live_collector/display.py | 8 +- .../sampling/live_collector/widgets.py | 56 ++++---- ...-12-06-19-49-20.gh-issue-138122.m3EF9E.rst | 5 + 6 files changed, 216 insertions(+), 86 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-12-06-19-49-20.gh-issue-138122.m3EF9E.rst diff --git a/Lib/_colorize.py b/Lib/_colorize.py index 0b7047620b4556..5c4903f14aa86b 100644 --- a/Lib/_colorize.py +++ b/Lib/_colorize.py @@ -9,7 +9,7 @@ # types if False: - from typing import IO, Self, ClassVar + from typing import IO, Literal, Self, ClassVar _theme: Theme @@ -74,6 +74,19 @@ class ANSIColors: setattr(NoColors, attr, "") +class CursesColors: + """Curses color constants for terminal UI theming.""" + BLACK = 0 + RED = 1 + GREEN = 2 + YELLOW = 3 + BLUE = 4 + MAGENTA = 5 + CYAN = 6 + WHITE = 7 + DEFAULT = -1 + + # # Experimental theming support (see gh-133346) # @@ -187,6 +200,114 @@ class Difflib(ThemeSection): reset: str = ANSIColors.RESET +@dataclass(frozen=True, kw_only=True) +class LiveProfiler(ThemeSection): + """Theme section for the live profiling TUI (Tachyon profiler). + + Colors use CursesColors constants (BLACK, RED, GREEN, YELLOW, + BLUE, MAGENTA, CYAN, WHITE, DEFAULT). + """ + # Header colors + title_fg: int = CursesColors.CYAN + title_bg: int = CursesColors.DEFAULT + + # Status display colors + pid_fg: int = CursesColors.CYAN + uptime_fg: int = CursesColors.GREEN + time_fg: int = CursesColors.YELLOW + interval_fg: int = CursesColors.MAGENTA + + # Thread view colors + thread_all_fg: int = CursesColors.GREEN + thread_single_fg: int = CursesColors.MAGENTA + + # Progress bar colors + bar_good_fg: int = CursesColors.GREEN + bar_bad_fg: int = CursesColors.RED + + # Stats colors + on_gil_fg: int = CursesColors.GREEN + off_gil_fg: int = CursesColors.RED + waiting_gil_fg: int = CursesColors.YELLOW + gc_fg: int = CursesColors.MAGENTA + + # Function display colors + func_total_fg: int = CursesColors.CYAN + func_exec_fg: int = CursesColors.GREEN + func_stack_fg: int = CursesColors.YELLOW + func_shown_fg: int = CursesColors.MAGENTA + + # Table header colors (for sorted column highlight) + sorted_header_fg: int = CursesColors.BLACK + sorted_header_bg: int = CursesColors.CYAN + + # Normal header colors (non-sorted columns) - use reverse video style + normal_header_fg: int = CursesColors.BLACK + normal_header_bg: int = CursesColors.WHITE + + # Data row colors + samples_fg: int = CursesColors.CYAN + file_fg: int = CursesColors.GREEN + func_fg: int = CursesColors.YELLOW + + # Trend indicator colors + trend_up_fg: int = CursesColors.GREEN + trend_down_fg: int = CursesColors.RED + + # Medal colors for top functions + medal_gold_fg: int = CursesColors.RED + medal_silver_fg: int = CursesColors.YELLOW + medal_bronze_fg: int = CursesColors.GREEN + + # Background style: 'dark' or 'light' + background_style: Literal["dark", "light"] = "dark" + + +LiveProfilerLight = LiveProfiler( + # Header colors + title_fg=CursesColors.BLUE, # Blue is more readable than cyan on light bg + + # Status display colors - darker colors for light backgrounds + pid_fg=CursesColors.BLUE, + uptime_fg=CursesColors.BLACK, + time_fg=CursesColors.BLACK, + interval_fg=CursesColors.BLUE, + + # Thread view colors + thread_all_fg=CursesColors.BLACK, + thread_single_fg=CursesColors.BLUE, + + # Stats colors + waiting_gil_fg=CursesColors.RED, + gc_fg=CursesColors.BLUE, + + # Function display colors + func_total_fg=CursesColors.BLUE, + func_exec_fg=CursesColors.BLACK, + func_stack_fg=CursesColors.BLACK, + func_shown_fg=CursesColors.BLUE, + + # Table header colors (for sorted column highlight) + sorted_header_fg=CursesColors.WHITE, + sorted_header_bg=CursesColors.BLUE, + + # Normal header colors (non-sorted columns) + normal_header_fg=CursesColors.WHITE, + normal_header_bg=CursesColors.BLACK, + + # Data row colors - use dark colors readable on white + samples_fg=CursesColors.BLACK, + file_fg=CursesColors.BLACK, + func_fg=CursesColors.BLUE, # Blue is more readable than magenta on light bg + + # Medal colors for top functions + medal_silver_fg=CursesColors.BLUE, + + # Background style + background_style="light", +) + + @dataclass(frozen=True, kw_only=True) class Syntax(ThemeSection): prompt: str = ANSIColors.BOLD_MAGENTA @@ -232,6 +353,7 @@ class Theme: """ argparse: Argparse = field(default_factory=Argparse) difflib: Difflib = field(default_factory=Difflib) + live_profiler: LiveProfiler = field(default_factory=LiveProfiler) syntax: Syntax = field(default_factory=Syntax) traceback: Traceback = field(default_factory=Traceback) unittest: Unittest = field(default_factory=Unittest) @@ -241,6 +363,7 @@ def copy_with( *, argparse: Argparse | None = None, difflib: Difflib | None = None, + live_profiler: LiveProfiler | None = None, syntax: Syntax | None = None, traceback: Traceback | None = None, unittest: Unittest | None = None, @@ -253,6 +376,7 @@ def copy_with( return type(self)( argparse=argparse or self.argparse, difflib=difflib or self.difflib, + live_profiler=live_profiler or self.live_profiler, syntax=syntax or self.syntax, traceback=traceback or self.traceback, unittest=unittest or self.unittest, @@ -269,6 +393,7 @@ def no_colors(cls) -> Self: return cls( argparse=Argparse.no_colors(), difflib=Difflib.no_colors(), + live_profiler=LiveProfiler.no_colors(), syntax=Syntax.no_colors(), traceback=Traceback.no_colors(), unittest=Unittest.no_colors(), @@ -338,6 +463,9 @@ def _safe_getenv(k: str, fallback: str | None = None) -> str | None: default_theme = Theme() theme_no_color = default_theme.no_colors() +# Convenience theme with light profiler colors (for white/light terminal backgrounds) +light_profiler_theme = default_theme.copy_with(live_profiler=LiveProfilerLight) + def get_theme( *, diff --git a/Lib/profiling/sampling/live_collector/collector.py b/Lib/profiling/sampling/live_collector/collector.py index b31ab060a6b934..c91ed9e0ea9367 100644 --- a/Lib/profiling/sampling/live_collector/collector.py +++ b/Lib/profiling/sampling/live_collector/collector.py @@ -33,6 +33,9 @@ FINISHED_BANNER_EXTRA_LINES, DEFAULT_SORT_BY, DEFAULT_DISPLAY_LIMIT, + COLOR_PAIR_SAMPLES, + COLOR_PAIR_FILE, + COLOR_PAIR_FUNC, COLOR_PAIR_HEADER_BG, COLOR_PAIR_CYAN, COLOR_PAIR_YELLOW, @@ -552,79 +555,61 @@ def _cycle_sort(self, reverse=False): def _setup_colors(self): """Set up color pairs and return color attributes.""" - A_BOLD = self.display.get_attr("A_BOLD") A_REVERSE = self.display.get_attr("A_REVERSE") A_UNDERLINE = self.display.get_attr("A_UNDERLINE") A_NORMAL = self.display.get_attr("A_NORMAL") - # Check both curses color support and _colorize.can_colorize() if self.display.has_colors() and self._can_colorize: with contextlib.suppress(Exception): - # Color constants (using curses values for compatibility) - COLOR_CYAN = 6 - COLOR_GREEN = 2 - COLOR_YELLOW = 3 - COLOR_BLACK = 0 - COLOR_MAGENTA = 5 - COLOR_RED = 1 - - # Initialize all color pairs used throughout the UI - self.display.init_color_pair( - 1, COLOR_CYAN, -1 - ) # Data colors for stats rows - self.display.init_color_pair(2, COLOR_GREEN, -1) - self.display.init_color_pair(3, COLOR_YELLOW, -1) - self.display.init_color_pair( - COLOR_PAIR_HEADER_BG, COLOR_BLACK, COLOR_GREEN - ) - self.display.init_color_pair( - COLOR_PAIR_CYAN, COLOR_CYAN, COLOR_BLACK - ) - self.display.init_color_pair( - COLOR_PAIR_YELLOW, COLOR_YELLOW, COLOR_BLACK - ) - self.display.init_color_pair( - COLOR_PAIR_GREEN, COLOR_GREEN, COLOR_BLACK - ) - self.display.init_color_pair( - COLOR_PAIR_MAGENTA, COLOR_MAGENTA, COLOR_BLACK - ) + theme = _colorize.get_theme(force_color=True).live_profiler + default_bg = -1 + + self.display.init_color_pair(COLOR_PAIR_SAMPLES, theme.samples_fg, default_bg) + self.display.init_color_pair(COLOR_PAIR_FILE, theme.file_fg, default_bg) + self.display.init_color_pair(COLOR_PAIR_FUNC, theme.func_fg, default_bg) + + # Normal header background color pair self.display.init_color_pair( - COLOR_PAIR_RED, COLOR_RED, COLOR_BLACK + COLOR_PAIR_HEADER_BG, + theme.normal_header_fg, + theme.normal_header_bg, ) + + self.display.init_color_pair(COLOR_PAIR_CYAN, theme.pid_fg, default_bg) + self.display.init_color_pair(COLOR_PAIR_YELLOW, theme.time_fg, default_bg) + self.display.init_color_pair(COLOR_PAIR_GREEN, theme.uptime_fg, default_bg) + self.display.init_color_pair(COLOR_PAIR_MAGENTA, theme.interval_fg, default_bg) + self.display.init_color_pair(COLOR_PAIR_RED, theme.off_gil_fg, default_bg) self.display.init_color_pair( - COLOR_PAIR_SORTED_HEADER, COLOR_BLACK, COLOR_YELLOW + COLOR_PAIR_SORTED_HEADER, + theme.sorted_header_fg, + theme.sorted_header_bg, ) + TREND_UP_PAIR = 11 + TREND_DOWN_PAIR = 12 + self.display.init_color_pair(TREND_UP_PAIR, theme.trend_up_fg, default_bg) + self.display.init_color_pair(TREND_DOWN_PAIR, theme.trend_down_fg, default_bg) + return { - "header": self.display.get_color_pair(COLOR_PAIR_HEADER_BG) - | A_BOLD, - "cyan": self.display.get_color_pair(COLOR_PAIR_CYAN) - | A_BOLD, - "yellow": self.display.get_color_pair(COLOR_PAIR_YELLOW) - | A_BOLD, - "green": self.display.get_color_pair(COLOR_PAIR_GREEN) - | A_BOLD, - "magenta": self.display.get_color_pair(COLOR_PAIR_MAGENTA) - | A_BOLD, - "red": self.display.get_color_pair(COLOR_PAIR_RED) - | A_BOLD, - "sorted_header": self.display.get_color_pair( - COLOR_PAIR_SORTED_HEADER - ) - | A_BOLD, - "normal_header": A_REVERSE | A_BOLD, - "color_samples": self.display.get_color_pair(1), - "color_file": self.display.get_color_pair(2), - "color_func": self.display.get_color_pair(3), - # Trend colors (stock-like indicators) - "trend_up": self.display.get_color_pair(COLOR_PAIR_GREEN) | A_BOLD, - "trend_down": self.display.get_color_pair(COLOR_PAIR_RED) | A_BOLD, + "header": self.display.get_color_pair(COLOR_PAIR_HEADER_BG) | A_BOLD, + "cyan": self.display.get_color_pair(COLOR_PAIR_CYAN) | A_BOLD, + "yellow": self.display.get_color_pair(COLOR_PAIR_YELLOW) | A_BOLD, + "green": self.display.get_color_pair(COLOR_PAIR_GREEN) | A_BOLD, + "magenta": self.display.get_color_pair(COLOR_PAIR_MAGENTA) | A_BOLD, + "red": self.display.get_color_pair(COLOR_PAIR_RED) | A_BOLD, + "sorted_header": self.display.get_color_pair(COLOR_PAIR_SORTED_HEADER) | A_BOLD, + "normal_header": self.display.get_color_pair(COLOR_PAIR_HEADER_BG) | A_BOLD, + "color_samples": self.display.get_color_pair(COLOR_PAIR_SAMPLES), + "color_file": self.display.get_color_pair(COLOR_PAIR_FILE), + "color_func": self.display.get_color_pair(COLOR_PAIR_FUNC), + "trend_up": self.display.get_color_pair(TREND_UP_PAIR) | A_BOLD, + "trend_down": self.display.get_color_pair(TREND_DOWN_PAIR) | A_BOLD, "trend_stable": A_NORMAL, } - # Fallback to non-color attributes + # Fallback for no-color mode return { "header": A_REVERSE | A_BOLD, "cyan": A_BOLD, @@ -637,7 +622,6 @@ def _setup_colors(self): "color_samples": A_NORMAL, "color_file": A_NORMAL, "color_func": A_NORMAL, - # Trend colors (fallback to bold/normal for monochrome) "trend_up": A_BOLD, "trend_down": A_BOLD, "trend_stable": A_NORMAL, diff --git a/Lib/profiling/sampling/live_collector/constants.py b/Lib/profiling/sampling/live_collector/constants.py index 4f4575f7b7aae2..bb45006553a67b 100644 --- a/Lib/profiling/sampling/live_collector/constants.py +++ b/Lib/profiling/sampling/live_collector/constants.py @@ -49,6 +49,9 @@ OPCODE_PANEL_HEIGHT = 12 # Height reserved for opcode statistics panel # Color pair IDs +COLOR_PAIR_SAMPLES = 1 +COLOR_PAIR_FILE = 2 +COLOR_PAIR_FUNC = 3 COLOR_PAIR_HEADER_BG = 4 COLOR_PAIR_CYAN = 5 COLOR_PAIR_YELLOW = 6 diff --git a/Lib/profiling/sampling/live_collector/display.py b/Lib/profiling/sampling/live_collector/display.py index d7f65ad73fdc6d..f5324421b10211 100644 --- a/Lib/profiling/sampling/live_collector/display.py +++ b/Lib/profiling/sampling/live_collector/display.py @@ -74,13 +74,17 @@ def get_dimensions(self): return self.stdscr.getmaxyx() def clear(self): - self.stdscr.clear() + # Use erase() instead of clear() to avoid flickering + # clear() forces a complete screen redraw, erase() just clears the buffer + self.stdscr.erase() def refresh(self): self.stdscr.refresh() def redraw(self): - self.stdscr.redrawwin() + # Use noutrefresh + doupdate for smoother updates + self.stdscr.noutrefresh() + curses.doupdate() def add_str(self, line, col, text, attr=0): try: diff --git a/Lib/profiling/sampling/live_collector/widgets.py b/Lib/profiling/sampling/live_collector/widgets.py index cf04f3aa3254ef..ac215dbfeb896e 100644 --- a/Lib/profiling/sampling/live_collector/widgets.py +++ b/Lib/profiling/sampling/live_collector/widgets.py @@ -641,8 +641,6 @@ def render(self, line, width, **kwargs): def draw_column_headers(self, line, width): """Draw column headers with sort indicators.""" - col = 0 - # Determine which columns to show based on width show_sample_pct = width >= WIDTH_THRESHOLD_SAMPLE_PCT show_tottime = width >= WIDTH_THRESHOLD_TOTTIME @@ -661,38 +659,38 @@ def draw_column_headers(self, line, width): "cumtime": 4, }.get(self.collector.sort_by, -1) + # Build the full header line first, then draw it + # This avoids gaps between columns when using reverse video + header_parts = [] + col = 0 + # Column 0: nsamples - attr = sorted_header if sort_col == 0 else normal_header - text = f"{'▼nsamples' if sort_col == 0 else 'nsamples':>13}" - self.add_str(line, col, text, attr) + text = f"{'▼nsamples' if sort_col == 0 else 'nsamples':>13} " + header_parts.append((col, text, sorted_header if sort_col == 0 else normal_header)) col += 15 # Column 1: sample % if show_sample_pct: - attr = sorted_header if sort_col == 1 else normal_header - text = f"{'▼%' if sort_col == 1 else '%':>5}" - self.add_str(line, col, text, attr) + text = f"{'▼%' if sort_col == 1 else '%':>5} " + header_parts.append((col, text, sorted_header if sort_col == 1 else normal_header)) col += 7 # Column 2: tottime if show_tottime: - attr = sorted_header if sort_col == 2 else normal_header - text = f"{'▼tottime' if sort_col == 2 else 'tottime':>10}" - self.add_str(line, col, text, attr) + text = f"{'▼tottime' if sort_col == 2 else 'tottime':>10} " + header_parts.append((col, text, sorted_header if sort_col == 2 else normal_header)) col += 12 # Column 3: cumul % if show_cumul_pct: - attr = sorted_header if sort_col == 3 else normal_header - text = f"{'▼%' if sort_col == 3 else '%':>5}" - self.add_str(line, col, text, attr) + text = f"{'▼%' if sort_col == 3 else '%':>5} " + header_parts.append((col, text, sorted_header if sort_col == 3 else normal_header)) col += 7 # Column 4: cumtime if show_cumtime: - attr = sorted_header if sort_col == 4 else normal_header - text = f"{'▼cumtime' if sort_col == 4 else 'cumtime':>10}" - self.add_str(line, col, text, attr) + text = f"{'▼cumtime' if sort_col == 4 else 'cumtime':>10} " + header_parts.append((col, text, sorted_header if sort_col == 4 else normal_header)) col += 12 # Remaining headers @@ -702,13 +700,22 @@ def draw_column_headers(self, line, width): MAX_FUNC_NAME_WIDTH, max(MIN_FUNC_NAME_WIDTH, remaining_space // 2), ) - self.add_str( - line, col, f"{'function':<{func_width}}", normal_header - ) + text = f"{'function':<{func_width}} " + header_parts.append((col, text, normal_header)) col += func_width + 2 if col < width - 10: - self.add_str(line, col, "file:line", normal_header) + file_text = "file:line" + padding = width - col - len(file_text) + text = file_text + " " * max(0, padding) + header_parts.append((col, text, normal_header)) + + # Draw full-width background first + self.add_str(line, 0, " " * (width - 1), normal_header) + + # Draw each header part on top + for col_pos, text, attr in header_parts: + self.add_str(line, col_pos, text.rstrip(), attr) return ( line + 1, @@ -724,8 +731,7 @@ def draw_stats_rows(self, line, height, width, stats_list, column_flags): column_flags ) - # Get color attributes from the colors dict (already initialized) - color_samples = self.colors.get("color_samples", curses.A_NORMAL) + # Get color attributes color_file = self.colors.get("color_file", curses.A_NORMAL) color_func = self.colors.get("color_func", curses.A_NORMAL) @@ -761,12 +767,12 @@ def draw_stats_rows(self, line, height, width, stats_list, column_flags): # Check if this row is selected is_selected = show_opcodes and row_idx == selected_row - # Helper function to get trend color for a specific column + # Helper function to get trend color def get_trend_color(column_name): if is_selected: return A_REVERSE | A_BOLD trend = trends.get(column_name, "stable") - if trend_tracker is not None: + if trend_tracker is not None and trend_tracker.enabled: return trend_tracker.get_color(trend) return curses.A_NORMAL diff --git a/Misc/NEWS.d/next/Library/2025-12-06-19-49-20.gh-issue-138122.m3EF9E.rst b/Misc/NEWS.d/next/Library/2025-12-06-19-49-20.gh-issue-138122.m3EF9E.rst new file mode 100644 index 00000000000000..f4e024828e2a7b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-06-19-49-20.gh-issue-138122.m3EF9E.rst @@ -0,0 +1,5 @@ +The Tachyon profiler's live TUI now integrates with the experimental +:mod:`!_colorize` theming system. Users can customize colors via +:func:`!_colorize.set_theme` (experimental API, subject to change). +A :class:`!LiveProfilerLight` theme is provided for light terminal backgrounds. +Patch by Pablo Galindo. From 6b9a6c6ec3bbc9795df67b87340e2ea58f42b3d4 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Fri, 2 Jan 2026 02:31:39 +0000 Subject: [PATCH 628/638] gh-138122: Move local imports to module level in sampling profiler (#143257) --- Lib/profiling/sampling/binary_reader.py | 13 +-- Lib/profiling/sampling/cli.py | 15 ++- Lib/profiling/sampling/heatmap_collector.py | 11 +-- .../sampling/live_collector/collector.py | 2 - .../sampling/live_collector/widgets.py | 3 +- Lib/profiling/sampling/pstats_collector.py | 6 +- Lib/profiling/sampling/stack_collector.py | 2 +- .../test_sampling_profiler/test_advanced.py | 6 +- .../test_sampling_profiler/test_async.py | 18 +--- .../test_sampling_profiler/test_children.py | 93 +++++++------------ .../test_sampling_profiler/test_collectors.py | 4 +- .../test_integration.py | 15 +-- .../test_live_collector_interaction.py | 8 +- .../test_sampling_profiler/test_modes.py | 23 ++--- .../test_sampling_profiler/test_profiler.py | 2 +- 15 files changed, 82 insertions(+), 139 deletions(-) diff --git a/Lib/profiling/sampling/binary_reader.py b/Lib/profiling/sampling/binary_reader.py index 50c96668cc585b..a11be3652597a6 100644 --- a/Lib/profiling/sampling/binary_reader.py +++ b/Lib/profiling/sampling/binary_reader.py @@ -1,5 +1,11 @@ """Thin Python wrapper around C binary reader for profiling data.""" +import _remote_debugging + +from .gecko_collector import GeckoCollector +from .stack_collector import FlamegraphCollector, CollapsedStackCollector +from .pstats_collector import PstatsCollector + class BinaryReader: """High-performance binary reader using C implementation. @@ -23,7 +29,6 @@ def __init__(self, filename): self._reader = None def __enter__(self): - import _remote_debugging self._reader = _remote_debugging.BinaryReader(self.filename) return self @@ -99,10 +104,6 @@ def convert_binary_to_format(input_file, output_file, output_format, Returns: int: Number of samples converted """ - from .gecko_collector import GeckoCollector - from .stack_collector import FlamegraphCollector, CollapsedStackCollector - from .pstats_collector import PStatsCollector - with BinaryReader(input_file) as reader: info = reader.get_info() interval = sample_interval_usec or info['sample_interval_us'] @@ -113,7 +114,7 @@ def convert_binary_to_format(input_file, output_file, output_format, elif output_format == 'collapsed': collector = CollapsedStackCollector(interval) elif output_format == 'pstats': - collector = PStatsCollector(interval) + collector = PstatsCollector(interval) elif output_format == 'gecko': collector = GeckoCollector(interval) else: diff --git a/Lib/profiling/sampling/cli.py b/Lib/profiling/sampling/cli.py index ea3926c9565809..c0dcda46fc29d3 100644 --- a/Lib/profiling/sampling/cli.py +++ b/Lib/profiling/sampling/cli.py @@ -36,6 +36,12 @@ SORT_MODE_NSAMPLES_CUMUL, ) +try: + from ._child_monitor import ChildProcessMonitor +except ImportError: + # _remote_debugging module not available on this platform (e.g., WASI) + ChildProcessMonitor = None + try: from .live_collector import LiveStatsCollector except ImportError: @@ -94,8 +100,6 @@ class CustomFormatter( } def _setup_child_monitor(args, parent_pid): - from ._child_monitor import ChildProcessMonitor - # Build CLI args for child profilers (excluding --subprocesses to avoid recursion) child_cli_args = _build_child_profiler_args(args) @@ -691,6 +695,11 @@ def _validate_args(args, parser): # --subprocesses is incompatible with --live if hasattr(args, 'subprocesses') and args.subprocesses: + if ChildProcessMonitor is None: + parser.error( + "--subprocesses is not available on this platform " + "(requires _remote_debugging module)." + ) if hasattr(args, 'live') and args.live: parser.error("--subprocesses is incompatible with --live mode.") @@ -1160,8 +1169,6 @@ def _handle_live_run(args): def _handle_replay(args): """Handle the 'replay' command - convert binary profile to another format.""" - import os - if not os.path.exists(args.input_file): sys.exit(f"Error: Input file not found: {args.input_file}") diff --git a/Lib/profiling/sampling/heatmap_collector.py b/Lib/profiling/sampling/heatmap_collector.py index 022e94d014f9b7..b6d9ff79e8ceec 100644 --- a/Lib/profiling/sampling/heatmap_collector.py +++ b/Lib/profiling/sampling/heatmap_collector.py @@ -18,6 +18,7 @@ from ._css_utils import get_combined_css from ._format_utils import fmt from .collector import normalize_location, extract_lineno +from .opcode_utils import get_opcode_info, format_opcode from .stack_collector import StackTraceCollector @@ -642,8 +643,6 @@ def _get_bytecode_data_for_line(self, filename, lineno): Returns: List of dicts with instruction info, sorted by samples descending """ - from .opcode_utils import get_opcode_info, format_opcode - key = (filename, lineno) opcode_data = self.line_opcodes.get(key, {}) @@ -1046,8 +1045,6 @@ def _render_source_with_highlights(self, line_content: str, line_num: int, Simple: collect ranges with sample counts, assign each byte position to smallest covering range, then emit spans for contiguous runs with sample data. """ - import html as html_module - content = line_content.rstrip('\n') if not content: return '' @@ -1070,7 +1067,7 @@ def _render_source_with_highlights(self, line_content: str, line_num: int, range_data[key]['opcodes'].append(opname) if not range_data: - return html_module.escape(content) + return html.escape(content) # For each byte position, find the smallest covering range byte_to_range = {} @@ -1098,7 +1095,7 @@ def _render_source_with_highlights(self, line_content: str, line_num: int, def flush_span(): nonlocal span_chars, current_range if span_chars: - text = html_module.escape(''.join(span_chars)) + text = html.escape(''.join(span_chars)) if current_range: data = range_data.get(current_range, {'samples': 0, 'opcodes': []}) samples = data['samples'] @@ -1112,7 +1109,7 @@ def flush_span(): f'data-samples="{samples}" ' f'data-max-samples="{max_range_samples}" ' f'data-pct="{pct}" ' - f'data-opcodes="{html_module.escape(opcodes)}">{text}') + f'data-opcodes="{html.escape(opcodes)}">{text}') else: result.append(text) span_chars = [] diff --git a/Lib/profiling/sampling/live_collector/collector.py b/Lib/profiling/sampling/live_collector/collector.py index c91ed9e0ea9367..c03df4075277cd 100644 --- a/Lib/profiling/sampling/live_collector/collector.py +++ b/Lib/profiling/sampling/live_collector/collector.py @@ -916,8 +916,6 @@ def _show_terminal_size_warning_and_wait(self, height, width): def _handle_input(self): """Handle keyboard input (non-blocking).""" - from . import constants - self.display.set_nodelay(True) ch = self.display.get_input() diff --git a/Lib/profiling/sampling/live_collector/widgets.py b/Lib/profiling/sampling/live_collector/widgets.py index ac215dbfeb896e..86d2649f875e62 100644 --- a/Lib/profiling/sampling/live_collector/widgets.py +++ b/Lib/profiling/sampling/live_collector/widgets.py @@ -31,6 +31,7 @@ PROFILING_MODE_GIL, PROFILING_MODE_WALL, ) +from ..opcode_utils import get_opcode_info, format_opcode class Widget(ABC): @@ -1013,8 +1014,6 @@ def render(self, line, width, **kwargs): Returns: Next available line number """ - from ..opcode_utils import get_opcode_info, format_opcode - stats_list = kwargs.get("stats_list", []) height = kwargs.get("height", 24) selected_row = self.collector.selected_row diff --git a/Lib/profiling/sampling/pstats_collector.py b/Lib/profiling/sampling/pstats_collector.py index e0dc9ab6bb7edb..6be1d698ffaa9a 100644 --- a/Lib/profiling/sampling/pstats_collector.py +++ b/Lib/profiling/sampling/pstats_collector.py @@ -1,9 +1,10 @@ import collections import marshal +import pstats from _colorize import ANSIColors from .collector import Collector, extract_lineno -from .constants import MICROSECONDS_PER_SECOND +from .constants import MICROSECONDS_PER_SECOND, PROFILING_MODE_CPU class PstatsCollector(Collector): @@ -86,9 +87,6 @@ def create_stats(self): def print_stats(self, sort=-1, limit=None, show_summary=True, mode=None): """Print formatted statistics to stdout.""" - import pstats - from .constants import PROFILING_MODE_CPU - # Create stats object stats = pstats.SampledStats(self).strip_dirs() if not stats.stats: diff --git a/Lib/profiling/sampling/stack_collector.py b/Lib/profiling/sampling/stack_collector.py index 55e643d0e9c8cb..4e213cfe41ca24 100644 --- a/Lib/profiling/sampling/stack_collector.py +++ b/Lib/profiling/sampling/stack_collector.py @@ -6,6 +6,7 @@ import linecache import os import sys +import sysconfig from ._css_utils import get_combined_css from .collector import Collector, extract_lineno @@ -244,7 +245,6 @@ def convert_children(children, min_samples): } # Calculate thread status percentages for display - import sysconfig is_free_threaded = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) total_threads = max(1, self.thread_status_counts["total"]) thread_stats = { diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_advanced.py b/Lib/test/test_profiling/test_sampling_profiler/test_advanced.py index bcd4de7f5d7ebe..11b1ad84242fd4 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_advanced.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_advanced.py @@ -11,6 +11,8 @@ import _remote_debugging # noqa: F401 import profiling.sampling import profiling.sampling.sample + from profiling.sampling.pstats_collector import PstatsCollector + from profiling.sampling.stack_collector import CollapsedStackCollector except ImportError: raise unittest.SkipTest( "Test only runs when _remote_debugging is available" @@ -61,7 +63,6 @@ def test_gc_frames_enabled(self): io.StringIO() as captured_output, mock.patch("sys.stdout", captured_output), ): - from profiling.sampling.pstats_collector import PstatsCollector collector = PstatsCollector(sample_interval_usec=5000, skip_idle=False) profiling.sampling.sample.sample( subproc.process.pid, @@ -88,7 +89,6 @@ def test_gc_frames_disabled(self): io.StringIO() as captured_output, mock.patch("sys.stdout", captured_output), ): - from profiling.sampling.pstats_collector import PstatsCollector collector = PstatsCollector(sample_interval_usec=5000, skip_idle=False) profiling.sampling.sample.sample( subproc.process.pid, @@ -140,7 +140,6 @@ def test_native_frames_enabled(self): io.StringIO() as captured_output, mock.patch("sys.stdout", captured_output), ): - from profiling.sampling.stack_collector import CollapsedStackCollector collector = CollapsedStackCollector(1000, skip_idle=False) profiling.sampling.sample.sample( subproc.process.pid, @@ -176,7 +175,6 @@ def test_native_frames_disabled(self): io.StringIO() as captured_output, mock.patch("sys.stdout", captured_output), ): - from profiling.sampling.pstats_collector import PstatsCollector collector = PstatsCollector(sample_interval_usec=5000, skip_idle=False) profiling.sampling.sample.sample( subproc.process.pid, diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_async.py b/Lib/test/test_profiling/test_sampling_profiler/test_async.py index d8ca86c996bffa..1f5685717b6273 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_async.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_async.py @@ -6,11 +6,14 @@ 3. Stack traversal: _build_linear_stacks() with BFS """ +import inspect import unittest try: import _remote_debugging # noqa: F401 from profiling.sampling.pstats_collector import PstatsCollector + from profiling.sampling.stack_collector import FlamegraphCollector + from profiling.sampling.sample import sample, sample_live, SampleProfiler except ImportError: raise unittest.SkipTest( "Test only runs when _remote_debugging is available" @@ -561,8 +564,6 @@ class TestFlamegraphCollectorAsync(unittest.TestCase): def test_flamegraph_with_async_frames(self): """Test FlamegraphCollector correctly processes async task frames.""" - from profiling.sampling.stack_collector import FlamegraphCollector - collector = FlamegraphCollector(sample_interval_usec=1000) # Build async task tree: Root -> Child @@ -607,8 +608,6 @@ def test_flamegraph_with_async_frames(self): def test_flamegraph_with_task_markers(self): """Test FlamegraphCollector includes boundary markers.""" - from profiling.sampling.stack_collector import FlamegraphCollector - collector = FlamegraphCollector(sample_interval_usec=1000) task = MockTaskInfo( @@ -643,8 +642,6 @@ def find_task_marker(node, depth=0): def test_flamegraph_multiple_async_samples(self): """Test FlamegraphCollector aggregates multiple async samples correctly.""" - from profiling.sampling.stack_collector import FlamegraphCollector - collector = FlamegraphCollector(sample_interval_usec=1000) task = MockTaskInfo( @@ -675,25 +672,16 @@ class TestAsyncAwareParameterFlow(unittest.TestCase): def test_sample_function_accepts_async_aware(self): """Test that sample() function accepts async_aware parameter.""" - from profiling.sampling.sample import sample - import inspect - sig = inspect.signature(sample) self.assertIn("async_aware", sig.parameters) def test_sample_live_function_accepts_async_aware(self): """Test that sample_live() function accepts async_aware parameter.""" - from profiling.sampling.sample import sample_live - import inspect - sig = inspect.signature(sample_live) self.assertIn("async_aware", sig.parameters) def test_sample_profiler_sample_accepts_async_aware(self): """Test that SampleProfiler.sample() accepts async_aware parameter.""" - from profiling.sampling.sample import SampleProfiler - import inspect - sig = inspect.signature(SampleProfiler.sample) self.assertIn("async_aware", sig.parameters) diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_children.py b/Lib/test/test_profiling/test_sampling_profiler/test_children.py index 84d50cd2088a9e..bb49faa890f348 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_children.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_children.py @@ -10,6 +10,7 @@ import threading import time import unittest +from unittest.mock import MagicMock, patch from test.support import ( SHORT_TIMEOUT, @@ -17,6 +18,40 @@ requires_remote_subprocess_debugging, ) +# Guard imports that require _remote_debugging module. +# This module is not available on all platforms (e.g., WASI). +try: + from profiling.sampling._child_monitor import ( + get_child_pids, + ChildProcessMonitor, + is_python_process, + _MAX_CHILD_PROFILERS, + _CLEANUP_INTERVAL_CYCLES, + ) +except ImportError: + # Module will be skipped via @requires_remote_subprocess_debugging decorators + get_child_pids = None + ChildProcessMonitor = None + is_python_process = None + _MAX_CHILD_PROFILERS = None + _CLEANUP_INTERVAL_CYCLES = None + +try: + from profiling.sampling.cli import ( + _add_sampling_options, + _validate_args, + _build_child_profiler_args, + _build_output_pattern, + _setup_child_monitor, + ) +except ImportError: + # cli module imports sample module which requires _remote_debugging + _add_sampling_options = None + _validate_args = None + _build_child_profiler_args = None + _build_output_pattern = None + _setup_child_monitor = None + from .helpers import _cleanup_process # String to check for in stderr when profiler lacks permissions (e.g., macOS) @@ -100,8 +135,6 @@ def test_get_child_pids_from_remote_debugging(self): def test_get_child_pids_fallback(self): """Test the fallback implementation for get_child_pids.""" - from profiling.sampling._child_monitor import get_child_pids - # Test with current process result = get_child_pids(os.getpid()) self.assertIsInstance(result, list) @@ -109,8 +142,6 @@ def test_get_child_pids_fallback(self): @unittest.skipUnless(sys.platform == "linux", "Linux only") def test_discover_child_process_linux(self): """Test that we can discover child processes on Linux.""" - from profiling.sampling._child_monitor import get_child_pids - # Create a child process proc = subprocess.Popen( [sys.executable, "-c", "import time; time.sleep(10)"], @@ -139,8 +170,6 @@ def test_discover_child_process_linux(self): def test_recursive_child_discovery(self): """Test that recursive=True finds grandchildren.""" - from profiling.sampling._child_monitor import get_child_pids - # Create a child that spawns a grandchild and keeps a reference to it # so we can clean it up via the child process code = """ @@ -256,8 +285,6 @@ def wait_for_signal(): def test_nonexistent_pid_returns_empty(self): """Test that nonexistent PID returns empty list.""" - from profiling.sampling._child_monitor import get_child_pids - # Use a very high PID that's unlikely to exist result = get_child_pids(999999999) self.assertEqual(result, []) @@ -275,8 +302,6 @@ def tearDown(self): def test_monitor_creation(self): """Test that ChildProcessMonitor can be created.""" - from profiling.sampling._child_monitor import ChildProcessMonitor - monitor = ChildProcessMonitor( pid=os.getpid(), cli_args=["-r", "10khz", "-d", "5"], @@ -288,8 +313,6 @@ def test_monitor_creation(self): def test_monitor_lifecycle(self): """Test monitor lifecycle via context manager.""" - from profiling.sampling._child_monitor import ChildProcessMonitor - monitor = ChildProcessMonitor( pid=os.getpid(), cli_args=[], output_pattern=None ) @@ -307,8 +330,6 @@ def test_monitor_lifecycle(self): def test_spawned_profilers_property(self): """Test that spawned_profilers returns a copy of the list.""" - from profiling.sampling._child_monitor import ChildProcessMonitor - monitor = ChildProcessMonitor( pid=os.getpid(), cli_args=[], output_pattern=None ) @@ -320,8 +341,6 @@ def test_spawned_profilers_property(self): def test_context_manager(self): """Test that ChildProcessMonitor works as a context manager.""" - from profiling.sampling._child_monitor import ChildProcessMonitor - with ChildProcessMonitor( pid=os.getpid(), cli_args=[], output_pattern=None ) as monitor: @@ -344,8 +363,6 @@ def tearDown(self): def test_subprocesses_flag_parsed(self): """Test that --subprocesses flag is recognized.""" - from profiling.sampling.cli import _add_sampling_options - parser = argparse.ArgumentParser() _add_sampling_options(parser) @@ -359,8 +376,6 @@ def test_subprocesses_flag_parsed(self): def test_subprocesses_incompatible_with_live(self): """Test that --subprocesses is incompatible with --live.""" - from profiling.sampling.cli import _validate_args - # Create mock args with both subprocesses and live args = argparse.Namespace( subprocesses=True, @@ -383,8 +398,6 @@ def test_subprocesses_incompatible_with_live(self): def test_build_child_profiler_args(self): """Test building CLI args for child profilers.""" - from profiling.sampling.cli import _build_child_profiler_args - args = argparse.Namespace( sample_interval_usec=200, duration=15, @@ -446,8 +459,6 @@ def assert_flag_value_pair(flag, value): def test_build_child_profiler_args_no_gc(self): """Test building CLI args with --no-gc.""" - from profiling.sampling.cli import _build_child_profiler_args - args = argparse.Namespace( sample_interval_usec=100, duration=5, @@ -471,8 +482,6 @@ def test_build_child_profiler_args_no_gc(self): def test_build_output_pattern_with_outfile(self): """Test output pattern generation with user-specified output.""" - from profiling.sampling.cli import _build_output_pattern - # With extension args = argparse.Namespace(outfile="output.html", format="flamegraph") pattern = _build_output_pattern(args) @@ -485,8 +494,6 @@ def test_build_output_pattern_with_outfile(self): def test_build_output_pattern_default(self): """Test output pattern generation with default output.""" - from profiling.sampling.cli import _build_output_pattern - # Flamegraph format args = argparse.Namespace(outfile=None, format="flamegraph") pattern = _build_output_pattern(args) @@ -512,8 +519,6 @@ def tearDown(self): def test_setup_child_monitor(self): """Test setting up a child monitor from args.""" - from profiling.sampling.cli import _setup_child_monitor - args = argparse.Namespace( sample_interval_usec=100, duration=5, @@ -553,8 +558,6 @@ def tearDown(self): def test_is_python_process_current_process(self): """Test that current process is detected as Python.""" - from profiling.sampling._child_monitor import is_python_process - # Current process should be Python result = is_python_process(os.getpid()) self.assertTrue( @@ -564,8 +567,6 @@ def test_is_python_process_current_process(self): def test_is_python_process_python_subprocess(self): """Test that a Python subprocess is detected as Python.""" - from profiling.sampling._child_monitor import is_python_process - # Start a Python subprocess proc = subprocess.Popen( [sys.executable, "-c", "import time; time.sleep(10)"], @@ -598,8 +599,6 @@ def test_is_python_process_python_subprocess(self): @unittest.skipUnless(sys.platform == "linux", "Linux only test") def test_is_python_process_non_python_subprocess(self): """Test that a non-Python subprocess is not detected as Python.""" - from profiling.sampling._child_monitor import is_python_process - # Start a non-Python subprocess (sleep command) proc = subprocess.Popen( ["sleep", "10"], @@ -624,8 +623,6 @@ def test_is_python_process_non_python_subprocess(self): def test_is_python_process_nonexistent_pid(self): """Test that nonexistent PID returns False.""" - from profiling.sampling._child_monitor import is_python_process - # Use a very high PID that's unlikely to exist result = is_python_process(999999999) self.assertFalse( @@ -635,8 +632,6 @@ def test_is_python_process_nonexistent_pid(self): def test_is_python_process_exited_process(self): """Test handling of a process that exits quickly.""" - from profiling.sampling._child_monitor import is_python_process - # Start a process that exits immediately proc = subprocess.Popen( [sys.executable, "-c", "pass"], @@ -666,8 +661,6 @@ def tearDown(self): def test_max_profilers_constant_exists(self): """Test that _MAX_CHILD_PROFILERS constant is defined.""" - from profiling.sampling._child_monitor import _MAX_CHILD_PROFILERS - self.assertEqual( _MAX_CHILD_PROFILERS, 100, @@ -676,8 +669,6 @@ def test_max_profilers_constant_exists(self): def test_cleanup_interval_constant_exists(self): """Test that _CLEANUP_INTERVAL_CYCLES constant is defined.""" - from profiling.sampling._child_monitor import _CLEANUP_INTERVAL_CYCLES - self.assertEqual( _CLEANUP_INTERVAL_CYCLES, 10, @@ -686,12 +677,6 @@ def test_cleanup_interval_constant_exists(self): def test_monitor_respects_max_limit(self): """Test that monitor refuses to spawn more than _MAX_CHILD_PROFILERS.""" - from profiling.sampling._child_monitor import ( - ChildProcessMonitor, - _MAX_CHILD_PROFILERS, - ) - from unittest.mock import MagicMock, patch - # Create a monitor monitor = ChildProcessMonitor( pid=os.getpid(), @@ -744,8 +729,6 @@ def tearDown(self): def test_wait_for_profilers_empty_list(self): """Test that wait_for_profilers returns immediately with no profilers.""" - from profiling.sampling._child_monitor import ChildProcessMonitor - monitor = ChildProcessMonitor( pid=os.getpid(), cli_args=[], output_pattern=None ) @@ -772,8 +755,6 @@ def test_wait_for_profilers_empty_list(self): def test_wait_for_profilers_with_completed_process(self): """Test waiting for profilers that complete quickly.""" - from profiling.sampling._child_monitor import ChildProcessMonitor - monitor = ChildProcessMonitor( pid=os.getpid(), cli_args=[], output_pattern=None ) @@ -812,8 +793,6 @@ def test_wait_for_profilers_with_completed_process(self): def test_wait_for_profilers_timeout(self): """Test that wait_for_profilers respects timeout.""" - from profiling.sampling._child_monitor import ChildProcessMonitor - monitor = ChildProcessMonitor( pid=os.getpid(), cli_args=[], output_pattern=None ) @@ -852,8 +831,6 @@ def test_wait_for_profilers_timeout(self): def test_wait_for_profilers_multiple(self): """Test waiting for multiple profilers.""" - from profiling.sampling._child_monitor import ChildProcessMonitor - monitor = ChildProcessMonitor( pid=os.getpid(), cli_args=[], output_pattern=None ) diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py index 30615a7d31d86c..13bdb4e111364c 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py @@ -2,6 +2,7 @@ import json import marshal +import opcode import os import tempfile import unittest @@ -1437,7 +1438,6 @@ class TestOpcodeFormatting(unittest.TestCase): def test_get_opcode_info_standard_opcode(self): """Test get_opcode_info for a standard opcode.""" - import opcode # LOAD_CONST is a standard opcode load_const = opcode.opmap.get('LOAD_CONST') if load_const is not None: @@ -1455,7 +1455,6 @@ def test_get_opcode_info_unknown_opcode(self): def test_format_opcode_standard(self): """Test format_opcode for a standard opcode.""" - import opcode load_const = opcode.opmap.get('LOAD_CONST') if load_const is not None: formatted = format_opcode(load_const) @@ -1463,7 +1462,6 @@ def test_format_opcode_standard(self): def test_format_opcode_specialized(self): """Test format_opcode for a specialized opcode shows base in parens.""" - import opcode if not hasattr(opcode, '_specialized_opmap'): self.skipTest("No specialized opcodes in this Python version") if not hasattr(opcode, '_specializations'): diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_integration.py b/Lib/test/test_profiling/test_sampling_profiler/test_integration.py index b82474858ddd4a..c6731e956391a9 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_integration.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_integration.py @@ -18,6 +18,7 @@ from profiling.sampling.pstats_collector import PstatsCollector from profiling.sampling.stack_collector import CollapsedStackCollector from profiling.sampling.sample import SampleProfiler, _is_process_running + from profiling.sampling.cli import main except ImportError: raise unittest.SkipTest( "Test only runs when _remote_debugging is available" @@ -547,7 +548,6 @@ def test_sample_target_script(self): io.StringIO() as captured_output, mock.patch("sys.stdout", captured_output), ): - from profiling.sampling.cli import main main() output = captured_output.getvalue() @@ -585,7 +585,6 @@ def test_sample_target_module(self): # Change to temp directory so subprocess can find the module contextlib.chdir(tempdir.name), ): - from profiling.sampling.cli import main main() output = captured_output.getvalue() @@ -714,8 +713,7 @@ def test_live_incompatible_with_pstats_options(self): test_args = ["profiling.sampling.cli", "run", "--live"] + args + ["test.py"] with mock.patch("sys.argv", test_args): with self.assertRaises(SystemExit) as cm: - from profiling.sampling.cli import main - main() + main() self.assertNotEqual(cm.exception.code, 0) def test_live_incompatible_with_multiple_pstats_options(self): @@ -727,8 +725,7 @@ def test_live_incompatible_with_multiple_pstats_options(self): with mock.patch("sys.argv", test_args): with self.assertRaises(SystemExit) as cm: - from profiling.sampling.cli import main - main() + main() self.assertNotEqual(cm.exception.code, 0) def test_live_incompatible_with_pstats_default_values(self): @@ -738,8 +735,7 @@ def test_live_incompatible_with_pstats_default_values(self): with mock.patch("sys.argv", test_args): with self.assertRaises(SystemExit) as cm: - from profiling.sampling.cli import main - main() + main() self.assertNotEqual(cm.exception.code, 0) # Test with --limit=15 (the default value) @@ -747,8 +743,7 @@ def test_live_incompatible_with_pstats_default_values(self): with mock.patch("sys.argv", test_args): with self.assertRaises(SystemExit) as cm: - from profiling.sampling.cli import main - main() + main() self.assertNotEqual(cm.exception.code, 0) diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_interaction.py b/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_interaction.py index 38f1d03e4939f1..8342faffb94762 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_interaction.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_interaction.py @@ -383,10 +383,9 @@ def test_finished_state_footer_message(self): def test_finished_state_freezes_time(self): """Test that time displays are frozen when finished.""" - import time as time_module # Set up collector with known start time - self.collector.start_time = time_module.perf_counter() - 10.0 # 10 seconds ago + self.collector.start_time = time.perf_counter() - 10.0 # 10 seconds ago # Mark as finished - this should freeze the time self.collector.mark_finished() @@ -396,7 +395,7 @@ def test_finished_state_freezes_time(self): frozen_time_display = self.collector.current_time_display # Wait a bit to ensure time would advance - time_module.sleep(0.1) + time.sleep(0.1) # Time should remain frozen self.assertEqual(self.collector.elapsed_time, frozen_elapsed) @@ -1215,7 +1214,6 @@ def test_reset_blocked_when_finished(self): def test_time_display_fix_when_finished(self): """Test that time display shows correct frozen time when finished.""" - import time as time_module # Mark as finished to freeze time self.collector.mark_finished() @@ -1228,7 +1226,7 @@ def test_time_display_fix_when_finished(self): frozen_time = self.collector.current_time_display # Wait a bit - time_module.sleep(0.1) + time.sleep(0.1) # Should still show the same frozen time (not jump to wrong time) self.assertEqual(self.collector.current_time_display, frozen_time) diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_modes.py b/Lib/test/test_profiling/test_sampling_profiler/test_modes.py index 877237866b1e65..0b38fb4ad4bcf6 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_modes.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_modes.py @@ -9,6 +9,12 @@ import profiling.sampling import profiling.sampling.sample from profiling.sampling.pstats_collector import PstatsCollector + from profiling.sampling.cli import main, _parse_mode + from profiling.sampling.constants import PROFILING_MODE_EXCEPTION + from _remote_debugging import ( + THREAD_STATUS_HAS_GIL, + THREAD_STATUS_ON_CPU, + ) except ImportError: raise unittest.SkipTest( "Test only runs when _remote_debugging is available" @@ -40,7 +46,6 @@ def test_mode_validation(self): mock.patch("sys.stderr", io.StringIO()) as mock_stderr, self.assertRaises(SystemExit) as cm, ): - from profiling.sampling.cli import main main() self.assertEqual(cm.exception.code, 2) # argparse error @@ -49,16 +54,6 @@ def test_mode_validation(self): def test_frames_filtered_with_skip_idle(self): """Test that frames are actually filtered when skip_idle=True.""" - # Import thread status flags - try: - from _remote_debugging import ( - THREAD_STATUS_HAS_GIL, - THREAD_STATUS_ON_CPU, - ) - except ImportError: - THREAD_STATUS_HAS_GIL = 1 << 0 - THREAD_STATUS_ON_CPU = 1 << 1 - # Create mock frames with different thread statuses class MockThreadInfoWithStatus: def __init__(self, thread_id, frame_info, status): @@ -240,7 +235,6 @@ class TestGilModeFiltering(unittest.TestCase): def test_gil_mode_validation(self): """Test that CLI accepts gil mode choice correctly.""" - from profiling.sampling.cli import main test_args = [ "profiling.sampling.cli", @@ -298,7 +292,6 @@ def test_gil_mode_sample_function_call(self): def test_gil_mode_cli_argument_parsing(self): """Test CLI argument parsing for GIL mode with various options.""" - from profiling.sampling.cli import main test_args = [ "profiling.sampling.cli", @@ -405,7 +398,6 @@ def test_mode_constants_are_defined(self): def test_parse_mode_function(self): """Test the _parse_mode function with all valid modes.""" - from profiling.sampling.cli import _parse_mode self.assertEqual(_parse_mode("wall"), 0) self.assertEqual(_parse_mode("cpu"), 1) self.assertEqual(_parse_mode("gil"), 2) @@ -422,7 +414,6 @@ class TestExceptionModeFiltering(unittest.TestCase): def test_exception_mode_validation(self): """Test that CLI accepts exception mode choice correctly.""" - from profiling.sampling.cli import main test_args = [ "profiling.sampling.cli", @@ -480,7 +471,6 @@ def test_exception_mode_sample_function_call(self): def test_exception_mode_cli_argument_parsing(self): """Test CLI argument parsing for exception mode with various options.""" - from profiling.sampling.cli import main test_args = [ "profiling.sampling.cli", @@ -512,7 +502,6 @@ def test_exception_mode_cli_argument_parsing(self): def test_exception_mode_constants_are_defined(self): """Test that exception mode constant is properly defined.""" - from profiling.sampling.constants import PROFILING_MODE_EXCEPTION self.assertEqual(PROFILING_MODE_EXCEPTION, 4) def test_exception_mode_integration_filtering(self): diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py b/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py index 822f559561eb0a..8d70a1d2ef8cfc 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py @@ -1,6 +1,7 @@ """Tests for sampling profiler core functionality.""" import io +import re from unittest import mock import unittest @@ -591,7 +592,6 @@ def test_print_sampled_stats_sort_by_name(self): # Extract just the function names for comparison func_names = [] - import re for line in data_lines: # Function name is between the last ( and ), accounting for ANSI color codes From 33410abcc835ad0eec9caa7ac9e8544db6b4b02b Mon Sep 17 00:00:00 2001 From: Vemulakonda559 <42494818+Vemulakonda559@users.noreply.github.com> Date: Wed, 12 Nov 2025 07:34:10 +0530 Subject: [PATCH 629/638] Clarify stdout flush behavior for newline characters in print() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR adds a note to the print() documentation to clarify how Python’s stdout buffering works with newline (\n) characters inside a single print call. Motivation: Current documentation mentions that flush() is implied for writes containing newlines. However, it does not explain that Python flushes only after the entire write operation, not mid-string. This can confuse users coming from C, who expect a flush at each newline, and developers writing scripts that rely on immediate output for progress indicators or CLI feedback. What’s added: A .. note:: block explaining that stdout behavior depends on the environment (TTY vs redirected stdout). Guidance on explicitly flushing with flush=True or sys.stdout.flush(). Mention of python -u for unbuffered output. A short example demonstrating the behavior. Impact: Improves clarity for learners and developers. Aligns documentation with actual behavior across different environments. Not a behavior change — documentation-only PR. Related Issue: Addresses issue #141395 --- Doc/library/functions.rst | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 7635e65296537d..4664742cec868a 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1616,6 +1616,31 @@ are always available. They are listed here in alphabetical order. Output buffering is usually determined by *file*. However, if *flush* is true, the stream is forcibly flushed. +.. note:: + + In Python, printing a string containing newline characters does not automatically flush stdout. + Python performs buffering at the write/operation level, so newlines inside a single write + do not necessarily trigger an immediate flush. The exact timing of output may vary depending + on the environment: + + - When stdout is connected to a terminal (TTY), output is line-buffered and typically flushes + after the write completes. + - When stdout is redirected to a file or pipe, output may be fully buffered and not flush + until the buffer fills or flush is requested. + + For guaranteed immediate output, use ``flush=True`` or call ``sys.stdout.flush()`` explicitly. + Running Python with the ``-u`` flag also forces unbuffered output, which may be useful in + scripts requiring immediate writes. + + Example: + + .. code-block:: python + + from time import sleep + + print("Hello\nWorld", end='') # Both lines appear together on TTY + sleep(3) + print("Hi there!") .. versionchanged:: 3.3 Added the *flush* keyword argument. From 9dac9fe1235c01545845cf88156d003386567a3a Mon Sep 17 00:00:00 2001 From: Vemulakonda559 Date: Sat, 29 Nov 2025 21:58:18 +0530 Subject: [PATCH 630/638] Refactor stdout flushing note for clarity Removed duplicate text and improved clarity in the note about stdout flushing behavior. --- Doc/library/functions.rst | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 4664742cec868a..e194909e0382c9 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1618,33 +1618,34 @@ are always available. They are listed here in alphabetical order. .. note:: - In Python, printing a string containing newline characters does not automatically flush stdout. - Python performs buffering at the write/operation level, so newlines inside a single write - do not necessarily trigger an immediate flush. The exact timing of output may vary depending + In Python, printing a string containing newline characters does not automatically flush stdout. + Python performs buffering at the write/operation level, so newlines inside a single write + do not necessarily trigger an immediate flush. The exact timing of output may vary depending on the environment: - - When stdout is connected to a terminal (TTY), output is line-buffered and typically flushes + - When stdout is connected to a terminal (TTY), output is line-buffered and typically flushes after the write completes. - - When stdout is redirected to a file or pipe, output may be fully buffered and not flush + - When stdout is redirected to a file or pipe, output may be fully buffered and not flush until the buffer fills or flush is requested. - For guaranteed immediate output, use ``flush=True`` or call ``sys.stdout.flush()`` explicitly. - Running Python with the ``-u`` flag also forces unbuffered output, which may be useful in + For guaranteed immediate output, use ``flush=True`` or call ``sys.stdout.flush()`` explicitly. + Running Python with the ``-u`` flag also forces unbuffered output, which may be useful in scripts requiring immediate writes. Example: .. code-block:: python - from time import sleep - print("Hello\nWorld", end='') # Both lines appear together on TTY + # Whether the default end is a newline ('\\n') or any other character, + # Python performs a single write operation for the entire string. + # Therefore, newlines inside the string do not cause mid-string flushing. + print("Hello\nWorld") sleep(3) print("Hi there!") - .. versionchanged:: 3.3 - Added the *flush* keyword argument. - +.. versionchanged:: 3.3 + Added the *flush* keyword argument. .. class:: property(fget=None, fset=None, fdel=None, doc=None) From 5652bfe3fc65a44bff5849946e8ea14f0632ec8e Mon Sep 17 00:00:00 2001 From: Vemulakonda559 Date: Sat, 29 Nov 2025 22:12:03 +0530 Subject: [PATCH 631/638] Refactor output buffering documentation for clarity --- Doc/library/functions.rst | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index e194909e0382c9..39aac77c93a0d8 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1616,27 +1616,29 @@ are always available. They are listed here in alphabetical order. Output buffering is usually determined by *file*. However, if *flush* is true, the stream is forcibly flushed. +Output buffering is usually determined by *file*. However, if *flush* is true, +the stream is forcibly flushed. + .. note:: - In Python, printing a string containing newline characters does not automatically flush stdout. - Python performs buffering at the write/operation level, so newlines inside a single write - do not necessarily trigger an immediate flush. The exact timing of output may vary depending - on the environment: + In Python, printing a string containing newline characters does not automatically + flush stdout. Python performs buffering at the write/operation level, so newlines + inside a single write do not necessarily trigger an immediate flush. The exact + timing of output may vary depending on the environment: - - When stdout is connected to a terminal (TTY), output is line-buffered and typically flushes - after the write completes. - - When stdout is redirected to a file or pipe, output may be fully buffered and not flush - until the buffer fills or flush is requested. + - When stdout is connected to a terminal (TTY), output is line-buffered and + typically flushes after the write completes. + - When stdout is redirected to a file or pipe, output may be fully buffered and + not flush until the buffer fills or flush is requested. - For guaranteed immediate output, use ``flush=True`` or call ``sys.stdout.flush()`` explicitly. - Running Python with the ``-u`` flag also forces unbuffered output, which may be useful in - scripts requiring immediate writes. + For guaranteed immediate output, use ``flush=True`` or call + ``sys.stdout.flush()`` explicitly. Running Python with the ``-u`` flag also + forces unbuffered output, which may be useful in scripts requiring immediate writes. Example: .. code-block:: python from time import sleep - # Whether the default end is a newline ('\\n') or any other character, # Python performs a single write operation for the entire string. # Therefore, newlines inside the string do not cause mid-string flushing. @@ -1647,6 +1649,7 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.3 Added the *flush* keyword argument. + .. class:: property(fget=None, fset=None, fdel=None, doc=None) Return a property attribute. From 80f9d5510298910af239f340190b6abeff0bd59a Mon Sep 17 00:00:00 2001 From: Vemulakonda559 Date: Sat, 29 Nov 2025 22:36:44 +0530 Subject: [PATCH 632/638] Simplify output buffering explanation Removed redundant sentence about output buffering. --- Doc/library/functions.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 39aac77c93a0d8..3176732e8866ba 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1616,9 +1616,6 @@ are always available. They are listed here in alphabetical order. Output buffering is usually determined by *file*. However, if *flush* is true, the stream is forcibly flushed. -Output buffering is usually determined by *file*. However, if *flush* is true, -the stream is forcibly flushed. - .. note:: In Python, printing a string containing newline characters does not automatically From eeb80155bcdfa3453f56c6ec61105150ec2154de Mon Sep 17 00:00:00 2001 From: Vemulakonda559 Date: Sat, 29 Nov 2025 22:52:39 +0530 Subject: [PATCH 633/638] Clean up comments in functions.rst Removed unnecessary comments about string flushing in print. --- Doc/library/functions.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 3176732e8866ba..5931a375748470 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1636,9 +1636,6 @@ are always available. They are listed here in alphabetical order. .. code-block:: python from time import sleep - # Whether the default end is a newline ('\\n') or any other character, - # Python performs a single write operation for the entire string. - # Therefore, newlines inside the string do not cause mid-string flushing. print("Hello\nWorld") sleep(3) print("Hi there!") @@ -1646,7 +1643,6 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.3 Added the *flush* keyword argument. - .. class:: property(fget=None, fset=None, fdel=None, doc=None) Return a property attribute. From 661aca5fd4b91f3d0ce061b4782293bde237de2f Mon Sep 17 00:00:00 2001 From: Vemulakonda559 Date: Sat, 29 Nov 2025 23:13:28 +0530 Subject: [PATCH 634/638] Fix formatting and indentation in functions.rst --- Doc/library/functions.rst | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 5931a375748470..a990458e9fef6c 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1616,29 +1616,29 @@ are always available. They are listed here in alphabetical order. Output buffering is usually determined by *file*. However, if *flush* is true, the stream is forcibly flushed. -.. note:: + .. note:: - In Python, printing a string containing newline characters does not automatically - flush stdout. Python performs buffering at the write/operation level, so newlines - inside a single write do not necessarily trigger an immediate flush. The exact - timing of output may vary depending on the environment: + In Python, printing a string containing newline characters does not automatically + flush stdout. Python performs buffering at the write/operation level, so newlines + inside a single write do not necessarily trigger an immediate flush. The exact + timing of output may vary depending on the environment: - - When stdout is connected to a terminal (TTY), output is line-buffered and - typically flushes after the write completes. - - When stdout is redirected to a file or pipe, output may be fully buffered and - not flush until the buffer fills or flush is requested. + - When stdout is connected to a terminal (TTY), output is line-buffered and + typically flushes after the write completes. + - When stdout is redirected to a file or pipe, output may be fully buffered and + not flush until the buffer fills or flush is requested. - For guaranteed immediate output, use ``flush=True`` or call - ``sys.stdout.flush()`` explicitly. Running Python with the ``-u`` flag also - forces unbuffered output, which may be useful in scripts requiring immediate writes. + For guaranteed immediate output, use ``flush=True`` or call + ``sys.stdout.flush()`` explicitly. Running Python with the ``-u`` flag also + forces unbuffered output, which may be useful in scripts requiring immediate writes. - Example: + Example: - .. code-block:: python - from time import sleep - print("Hello\nWorld") - sleep(3) - print("Hi there!") + .. code-block:: python + from time import sleep + print("Hello\nWorld") + sleep(3) + print("Hi there!") .. versionchanged:: 3.3 Added the *flush* keyword argument. From 710f49aa2b600276d08bdb191ddd2c9f779f3b0f Mon Sep 17 00:00:00 2001 From: Vemulakonda559 Date: Sat, 29 Nov 2025 23:25:24 +0530 Subject: [PATCH 635/638] Refactor function documentation formatting Remove extra blank lines and fix indentation for versionchanged directive. --- Doc/library/functions.rst | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index a990458e9fef6c..b9b34074116147 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1553,7 +1553,6 @@ are always available. They are listed here in alphabetical order. length 1, return its single byte value. For example, ``ord(b'a')`` returns the integer ``97``. - .. function:: pow(base, exp, mod=None) Return *base* to the power *exp*; if *mod* is present, return *base* to the @@ -1595,7 +1594,6 @@ are always available. They are listed here in alphabetical order. Allow keyword arguments. Formerly, only positional arguments were supported. - .. function:: print(*objects, sep=' ', end='\n', file=None, flush=False) Print *objects* to the text stream *file*, separated by *sep* and followed @@ -1640,8 +1638,8 @@ are always available. They are listed here in alphabetical order. sleep(3) print("Hi there!") -.. versionchanged:: 3.3 - Added the *flush* keyword argument. + .. versionchanged:: 3.3 + Added the *flush* keyword argument. .. class:: property(fget=None, fset=None, fdel=None, doc=None) From a54c1f478dbf2b247cc2161d86bcaf471cad797c Mon Sep 17 00:00:00 2001 From: Vemulakonda559 Date: Sat, 29 Nov 2025 23:48:21 +0530 Subject: [PATCH 636/638] Reorganize output buffering description in functions.rst Reformat output buffering explanation for clarity. --- Doc/library/functions.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index b9b34074116147..894ccd74dce6ff 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1611,8 +1611,8 @@ are always available. They are listed here in alphabetical order. arguments are converted to text strings, :func:`print` cannot be used with binary mode file objects. For these, use ``file.write(...)`` instead. - Output buffering is usually determined by *file*. - However, if *flush* is true, the stream is forcibly flushed. + Output buffering is usually determined by *file*. However, if *flush* is + true, the stream is forcibly flushed. .. note:: @@ -1634,6 +1634,8 @@ are always available. They are listed here in alphabetical order. .. code-block:: python from time import sleep + # This call performs one write operation, so the newline inside the string + # does not trigger an immediate flush by itself. print("Hello\nWorld") sleep(3) print("Hi there!") From 14bf4541fcb9be7facaa782190b166b8f95177f1 Mon Sep 17 00:00:00 2001 From: Vemulakonda559 Date: Mon, 1 Dec 2025 19:52:52 +0530 Subject: [PATCH 637/638] Fix indentation in functions.rst example code --- Doc/library/functions.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 894ccd74dce6ff..e3c639bcd28c60 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1633,12 +1633,12 @@ are always available. They are listed here in alphabetical order. Example: .. code-block:: python - from time import sleep - # This call performs one write operation, so the newline inside the string - # does not trigger an immediate flush by itself. - print("Hello\nWorld") - sleep(3) - print("Hi there!") + from time import sleep + # This call performs one write operation, so the newline inside the string + # does not trigger an immediate flush by itself. + print("Hello\nWorld") + sleep(3) + print("Hi there!") .. versionchanged:: 3.3 Added the *flush* keyword argument. From 0ece9e02d3fc3e89025dc401fe8a383b43b649e2 Mon Sep 17 00:00:00 2001 From: Vemulakonda559 Date: Mon, 1 Dec 2025 20:05:06 +0530 Subject: [PATCH 638/638] Fix indentation in example code block --- Doc/library/functions.rst | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index e3c639bcd28c60..6708571d33aedd 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1633,12 +1633,13 @@ are always available. They are listed here in alphabetical order. Example: .. code-block:: python - from time import sleep - # This call performs one write operation, so the newline inside the string - # does not trigger an immediate flush by itself. - print("Hello\nWorld") - sleep(3) - print("Hi there!") + + from time import sleep + # This call performs one write operation, so the newline inside the string + # does not trigger an immediate flush by itself. + print("Hello\nWorld") + sleep(3) + print("Hi there!") .. versionchanged:: 3.3 Added the *flush* keyword argument.