From 43013f72f0aadc5ee428aa5bdf6d949b4e79779a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 15 Sep 2025 17:07:34 +0300 Subject: [PATCH 01/18] gh-138779: Use the dev_t converter for st_rdev (GH-138780) This allows to support device numbers larger than 2**63-1. --- .../Library/2025-09-11-11-09-28.gh-issue-138779.TNZnLr.rst | 3 +++ Modules/posixmodule.c | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-09-11-11-09-28.gh-issue-138779.TNZnLr.rst diff --git a/Misc/NEWS.d/next/Library/2025-09-11-11-09-28.gh-issue-138779.TNZnLr.rst b/Misc/NEWS.d/next/Library/2025-09-11-11-09-28.gh-issue-138779.TNZnLr.rst new file mode 100644 index 00000000000000..d54f21ffb89669 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-09-11-11-09-28.gh-issue-138779.TNZnLr.rst @@ -0,0 +1,3 @@ +Support device numbers larger than ``2**63-1`` for the +:attr:`~os.stat_result.st_rdev` field of the :class:`os.stat_result` +structure. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 5e735e86bdee9e..bba73c659dd168 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -2759,7 +2759,7 @@ _pystat_fromstructstat(PyObject *module, STRUCT_STAT *st) SET_ITEM(ST_BLOCKS_IDX, PyLong_FromLong((long)st->st_blocks)); #endif #ifdef HAVE_STRUCT_STAT_ST_RDEV - SET_ITEM(ST_RDEV_IDX, PyLong_FromLong((long)st->st_rdev)); + SET_ITEM(ST_RDEV_IDX, _PyLong_FromDev(st->st_rdev)); #endif #ifdef HAVE_STRUCT_STAT_ST_GEN SET_ITEM(ST_GEN_IDX, PyLong_FromLong((long)st->st_gen)); From f07ae274b8e98c570d40b1aabd4cc42cb44a13ca Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 15 Sep 2025 15:21:43 +0100 Subject: [PATCH 02/18] gh-129813, PEP 782: Use PyBytesWriter in fcntl (#138921) Replace PyBytes_FromStringAndSize(NULL, size) with the new public PyBytesWriter API. Don't build the fcntl with the limited C API anymore, since the PyBytesWriter API is not part of the limited C API. --- Modules/clinic/fcntlmodule.c.h | 38 +++++++++++----------------------- Modules/fcntlmodule.c | 31 ++++++++++++++------------- 2 files changed, 27 insertions(+), 42 deletions(-) diff --git a/Modules/clinic/fcntlmodule.c.h b/Modules/clinic/fcntlmodule.c.h index 005e9b9e12afd9..2b61d9f87083f0 100644 --- a/Modules/clinic/fcntlmodule.c.h +++ b/Modules/clinic/fcntlmodule.c.h @@ -2,6 +2,8 @@ preserve [clinic start generated code]*/ +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + PyDoc_STRVAR(fcntl_fcntl__doc__, "fcntl($module, fd, cmd, arg=0, /)\n" "--\n" @@ -19,7 +21,7 @@ PyDoc_STRVAR(fcntl_fcntl__doc__, "corresponding to the return value of the fcntl call in the C code."); #define FCNTL_FCNTL_METHODDEF \ - {"fcntl", (PyCFunction)(void(*)(void))fcntl_fcntl, METH_FASTCALL, fcntl_fcntl__doc__}, + {"fcntl", _PyCFunction_CAST(fcntl_fcntl), METH_FASTCALL, fcntl_fcntl__doc__}, static PyObject * fcntl_fcntl_impl(PyObject *module, int fd, int code, PyObject *arg); @@ -32,12 +34,7 @@ fcntl_fcntl(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int code; PyObject *arg = NULL; - if (nargs < 2) { - PyErr_Format(PyExc_TypeError, "fcntl expected at least 2 arguments, got %zd", nargs); - goto exit; - } - if (nargs > 3) { - PyErr_Format(PyExc_TypeError, "fcntl expected at most 3 arguments, got %zd", nargs); + if (!_PyArg_CheckPositional("fcntl", nargs, 2, 3)) { goto exit; } fd = PyObject_AsFileDescriptor(args[0]); @@ -93,7 +90,7 @@ PyDoc_STRVAR(fcntl_ioctl__doc__, "code."); #define FCNTL_IOCTL_METHODDEF \ - {"ioctl", (PyCFunction)(void(*)(void))fcntl_ioctl, METH_FASTCALL, fcntl_ioctl__doc__}, + {"ioctl", _PyCFunction_CAST(fcntl_ioctl), METH_FASTCALL, fcntl_ioctl__doc__}, static PyObject * fcntl_ioctl_impl(PyObject *module, int fd, unsigned long code, PyObject *arg, @@ -108,12 +105,7 @@ fcntl_ioctl(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyObject *arg = NULL; int mutate_arg = 1; - if (nargs < 2) { - PyErr_Format(PyExc_TypeError, "ioctl expected at least 2 arguments, got %zd", nargs); - goto exit; - } - if (nargs > 4) { - PyErr_Format(PyExc_TypeError, "ioctl expected at most 4 arguments, got %zd", nargs); + if (!_PyArg_CheckPositional("ioctl", nargs, 2, 4)) { goto exit; } fd = PyObject_AsFileDescriptor(args[0]); @@ -121,7 +113,7 @@ fcntl_ioctl(PyObject *module, PyObject *const *args, Py_ssize_t nargs) goto exit; } if (!PyIndex_Check(args[1])) { - PyErr_Format(PyExc_TypeError, "ioctl() argument 2 must be int, not %T", args[1]); + _PyArg_BadArgument("ioctl", "argument 2", "int", args[1]); goto exit; } { @@ -168,7 +160,7 @@ PyDoc_STRVAR(fcntl_flock__doc__, "function is emulated using fcntl())."); #define FCNTL_FLOCK_METHODDEF \ - {"flock", (PyCFunction)(void(*)(void))fcntl_flock, METH_FASTCALL, fcntl_flock__doc__}, + {"flock", _PyCFunction_CAST(fcntl_flock), METH_FASTCALL, fcntl_flock__doc__}, static PyObject * fcntl_flock_impl(PyObject *module, int fd, int code); @@ -180,8 +172,7 @@ fcntl_flock(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int fd; int code; - if (nargs != 2) { - PyErr_Format(PyExc_TypeError, "flock expected 2 arguments, got %zd", nargs); + if (!_PyArg_CheckPositional("flock", nargs, 2, 2)) { goto exit; } fd = PyObject_AsFileDescriptor(args[0]); @@ -226,7 +217,7 @@ PyDoc_STRVAR(fcntl_lockf__doc__, " 2 - relative to the end of the file (SEEK_END)"); #define FCNTL_LOCKF_METHODDEF \ - {"lockf", (PyCFunction)(void(*)(void))fcntl_lockf, METH_FASTCALL, fcntl_lockf__doc__}, + {"lockf", _PyCFunction_CAST(fcntl_lockf), METH_FASTCALL, fcntl_lockf__doc__}, static PyObject * fcntl_lockf_impl(PyObject *module, int fd, int code, PyObject *lenobj, @@ -242,12 +233,7 @@ fcntl_lockf(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyObject *startobj = NULL; int whence = 0; - if (nargs < 2) { - PyErr_Format(PyExc_TypeError, "lockf expected at least 2 arguments, got %zd", nargs); - goto exit; - } - if (nargs > 5) { - PyErr_Format(PyExc_TypeError, "lockf expected at most 5 arguments, got %zd", nargs); + if (!_PyArg_CheckPositional("lockf", nargs, 2, 5)) { goto exit; } fd = PyObject_AsFileDescriptor(args[0]); @@ -279,4 +265,4 @@ fcntl_lockf(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=bf84289b741e7cf6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9773e44da302dc7c input=a9049054013a1b77]*/ diff --git a/Modules/fcntlmodule.c b/Modules/fcntlmodule.c index e49bf81b61f3be..df2c9994127997 100644 --- a/Modules/fcntlmodule.c +++ b/Modules/fcntlmodule.c @@ -1,9 +1,8 @@ /* fcntl module */ -// Need limited C API version 3.14 for PyLong_AsNativeBytes() in AC code -#include "pyconfig.h" // Py_GIL_DISABLED -#ifndef Py_GIL_DISABLED -# define Py_LIMITED_API 0x030e0000 +// Argument Clinic uses the internal C API +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 #endif #include "Python.h" @@ -113,12 +112,12 @@ fcntl_fcntl_impl(PyObject *module, int fd, int code, PyObject *arg) return PyBytes_FromStringAndSize(buf, len); } else { - PyObject *result = PyBytes_FromStringAndSize(NULL, len); - if (result == NULL) { + PyBytesWriter *writer = PyBytesWriter_Create(len); + if (writer == NULL) { PyBuffer_Release(&view); return NULL; } - char *ptr = PyBytes_AsString(result); + char *ptr = PyBytesWriter_GetData(writer); memcpy(ptr, view.buf, len); PyBuffer_Release(&view); @@ -131,15 +130,15 @@ fcntl_fcntl_impl(PyObject *module, int fd, int code, PyObject *arg) if (!async_err) { PyErr_SetFromErrno(PyExc_OSError); } - Py_DECREF(result); + PyBytesWriter_Discard(writer); return NULL; } if (ptr[len] != '\0') { PyErr_SetString(PyExc_SystemError, "buffer overflow"); - Py_DECREF(result); + PyBytesWriter_Discard(writer); return NULL; } - return result; + return PyBytesWriter_Finish(writer); } #undef FCNTL_BUFSZ } @@ -297,12 +296,12 @@ fcntl_ioctl_impl(PyObject *module, int fd, unsigned long code, PyObject *arg, return PyBytes_FromStringAndSize(buf, len); } else { - PyObject *result = PyBytes_FromStringAndSize(NULL, len); - if (result == NULL) { + PyBytesWriter *writer = PyBytesWriter_Create(len); + if (writer == NULL) { PyBuffer_Release(&view); return NULL; } - char *ptr = PyBytes_AsString(result); + char *ptr = PyBytesWriter_GetData(writer); memcpy(ptr, view.buf, len); PyBuffer_Release(&view); @@ -315,15 +314,15 @@ fcntl_ioctl_impl(PyObject *module, int fd, unsigned long code, PyObject *arg, if (!async_err) { PyErr_SetFromErrno(PyExc_OSError); } - Py_DECREF(result); + PyBytesWriter_Discard(writer); return NULL; } if (ptr[len] != '\0') { PyErr_SetString(PyExc_SystemError, "buffer overflow"); - Py_DECREF(result); + PyBytesWriter_Discard(writer); return NULL; } - return result; + return PyBytesWriter_Finish(writer); } #undef IOCTL_BUFSZ } From 7c6efc3a4f41f527ec40b5f5fd0ee1fb7af7c82f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 15 Sep 2025 15:23:11 +0100 Subject: [PATCH 03/18] gh-129813, PEP 782: Init small_buffer in PyBytesWriter_Create() (#138924) Fill small_buffer with 0xFF byte pattern to detect the usage of uninitialized bytes in debug build. --- Objects/bytesobject.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 3de57fe4e99e86..ee10f13b7bb04c 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3861,6 +3861,9 @@ byteswriter_create(Py_ssize_t size, int use_bytearray) return NULL; } } +#ifdef Py_DEBUG + memset(writer->small_buffer, 0xff, sizeof(writer->small_buffer)); +#endif writer->obj = NULL; writer->size = 0; writer->use_bytearray = use_bytearray; From 21c80cadc840265db533a2bdd07f717716209fde Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 15 Sep 2025 15:24:34 +0100 Subject: [PATCH 04/18] gh-129813, PEP 782: Use PyBytesWriter in _curses (#138920) Replace PyBytes_FromStringAndSize(NULL, size) and _PyBytes_Resize() with the new public PyBytesWriter API. --- Modules/_cursesmodule.c | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index 232dbcace9ac57..61464348d6fab8 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -1932,7 +1932,6 @@ PyCursesWindow_getstr(PyObject *op, PyObject *args) int rtn, use_xy = 0, y = 0, x = 0; unsigned int max_buf_size = 2048; unsigned int n = max_buf_size - 1; - PyObject *res; if (!curses_clinic_parse_optional_xy_n(args, &y, &x, &n, &use_xy, "_curses.window.instr")) @@ -1941,11 +1940,11 @@ PyCursesWindow_getstr(PyObject *op, PyObject *args) } n = Py_MIN(n, max_buf_size - 1); - res = PyBytes_FromStringAndSize(NULL, n + 1); - if (res == NULL) { + PyBytesWriter *writer = PyBytesWriter_Create(n + 1); + if (writer == NULL) { return NULL; } - char *buf = PyBytes_AS_STRING(res); + char *buf = PyBytesWriter_GetData(writer); if (use_xy) { Py_BEGIN_ALLOW_THREADS @@ -1965,11 +1964,10 @@ PyCursesWindow_getstr(PyObject *op, PyObject *args) } if (rtn == ERR) { - Py_DECREF(res); + PyBytesWriter_Discard(writer); return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); } - _PyBytes_Resize(&res, strlen(buf)); // 'res' is set to NULL on failure - return res; + return PyBytesWriter_FinishWithSize(writer, strlen(buf)); } /*[clinic input] @@ -2130,7 +2128,6 @@ PyCursesWindow_instr(PyObject *op, PyObject *args) int rtn, use_xy = 0, y = 0, x = 0; unsigned int max_buf_size = 2048; unsigned int n = max_buf_size - 1; - PyObject *res; if (!curses_clinic_parse_optional_xy_n(args, &y, &x, &n, &use_xy, "_curses.window.instr")) @@ -2139,11 +2136,11 @@ PyCursesWindow_instr(PyObject *op, PyObject *args) } n = Py_MIN(n, max_buf_size - 1); - res = PyBytes_FromStringAndSize(NULL, n + 1); - if (res == NULL) { + PyBytesWriter *writer = PyBytesWriter_Create(n + 1); + if (writer == NULL) { return NULL; } - char *buf = PyBytes_AS_STRING(res); + char *buf = PyBytesWriter_GetData(writer); if (use_xy) { rtn = mvwinnstr(self->win, y, x, buf, n); @@ -2153,11 +2150,10 @@ PyCursesWindow_instr(PyObject *op, PyObject *args) } if (rtn == ERR) { - Py_DECREF(res); + PyBytesWriter_Discard(writer); return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); } - _PyBytes_Resize(&res, strlen(buf)); // 'res' is set to NULL on failure - return res; + return PyBytesWriter_FinishWithSize(writer, strlen(buf)); } /*[clinic input] From 67cc1cf68a26d931ece3a6790ba914cf8a9b62f8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 15 Sep 2025 15:32:43 +0100 Subject: [PATCH 05/18] gh-129813, PEP 782: Use PyBytesWriter in _codecs.escape_decode() (#138919) Replace PyBytes_FromStringAndSize(NULL, size) and _PyBytes_Resize() with the new public PyBytesWriter API. --- Modules/_codecsmodule.c | 69 +++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 37 deletions(-) diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c index 82a46ec1e70d67..33e262f2ba1e65 100644 --- a/Modules/_codecsmodule.c +++ b/Modules/_codecsmodule.c @@ -202,55 +202,50 @@ _codecs_escape_encode_impl(PyObject *module, PyObject *data, const char *errors) /*[clinic end generated code: output=4af1d477834bab34 input=8f4b144799a94245]*/ { - Py_ssize_t size; - Py_ssize_t newsize; - PyObject *v; - - size = PyBytes_GET_SIZE(data); + Py_ssize_t size = PyBytes_GET_SIZE(data); if (size > PY_SSIZE_T_MAX / 4) { PyErr_SetString(PyExc_OverflowError, "string is too large to encode"); return NULL; } - newsize = 4*size; - v = PyBytes_FromStringAndSize(NULL, newsize); + Py_ssize_t newsize = 4*size; - if (v == NULL) { + PyBytesWriter *writer = PyBytesWriter_Create(newsize); + if (writer == NULL) { return NULL; } - else { - Py_ssize_t i; - char c; - char *p = PyBytes_AS_STRING(v); - - for (i = 0; i < size; i++) { - /* There's at least enough room for a hex escape */ - assert(newsize - (p - PyBytes_AS_STRING(v)) >= 4); - c = PyBytes_AS_STRING(data)[i]; - if (c == '\'' || c == '\\') - *p++ = '\\', *p++ = c; - else if (c == '\t') - *p++ = '\\', *p++ = 't'; - else if (c == '\n') - *p++ = '\\', *p++ = 'n'; - else if (c == '\r') - *p++ = '\\', *p++ = 'r'; - else if (c < ' ' || c >= 0x7f) { - *p++ = '\\'; - *p++ = 'x'; - *p++ = Py_hexdigits[(c & 0xf0) >> 4]; - *p++ = Py_hexdigits[c & 0xf]; - } - else - *p++ = c; + char *p = PyBytesWriter_GetData(writer); + + for (Py_ssize_t i = 0; i < size; i++) { + /* There's at least enough room for a hex escape */ + assert(newsize - (p - (char*)PyBytesWriter_GetData(writer)) >= 4); + + char c = PyBytes_AS_STRING(data)[i]; + if (c == '\'' || c == '\\') { + *p++ = '\\'; *p++ = c; } - *p = '\0'; - if (_PyBytes_Resize(&v, (p - PyBytes_AS_STRING(v)))) { - return NULL; + else if (c == '\t') { + *p++ = '\\'; *p++ = 't'; + } + else if (c == '\n') { + *p++ = '\\'; *p++ = 'n'; + } + else if (c == '\r') { + *p++ = '\\'; *p++ = 'r'; + } + else if (c < ' ' || c >= 0x7f) { + *p++ = '\\'; + *p++ = 'x'; + *p++ = Py_hexdigits[(c & 0xf0) >> 4]; + *p++ = Py_hexdigits[c & 0xf]; + } + else { + *p++ = c; } } - return codec_tuple(v, size); + PyObject *decoded = PyBytesWriter_FinishWithPointer(writer, p); + return codec_tuple(decoded, size); } /* --- Decoder ------------------------------------------------------------ */ From a5b9d0b8b273eaf7cfee8bb5770449b2e4395993 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Mon, 15 Sep 2025 15:36:17 +0100 Subject: [PATCH 06/18] gh-134953: Expand theming for `True`/`False`/`None` (#135000) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Łukasz Langa --- Lib/_colorize.py | 1 + Lib/_pyrepl/utils.py | 3 +++ .../Library/2025-06-01-11-14-00.gh-issue-134953.ashdfs.rst | 2 ++ 3 files changed, 6 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2025-06-01-11-14-00.gh-issue-134953.ashdfs.rst diff --git a/Lib/_colorize.py b/Lib/_colorize.py index 325efed274aed7..f45e7b8bb300f1 100644 --- a/Lib/_colorize.py +++ b/Lib/_colorize.py @@ -187,6 +187,7 @@ class Difflib(ThemeSection): class Syntax(ThemeSection): prompt: str = ANSIColors.BOLD_MAGENTA keyword: str = ANSIColors.BOLD_BLUE + keyword_constant: str = ANSIColors.BOLD_BLUE builtin: str = ANSIColors.CYAN comment: str = ANSIColors.RED string: str = ANSIColors.GREEN diff --git a/Lib/_pyrepl/utils.py b/Lib/_pyrepl/utils.py index c5d006afa7731f..d32fce591fadcc 100644 --- a/Lib/_pyrepl/utils.py +++ b/Lib/_pyrepl/utils.py @@ -196,6 +196,9 @@ def gen_colors_from_token_stream( is_def_name = False span = Span.from_token(token, line_lengths) yield ColorSpan(span, "definition") + elif token.string in ("True", "False", "None"): + span = Span.from_token(token, line_lengths) + yield ColorSpan(span, "keyword_constant") elif keyword.iskeyword(token.string): span = Span.from_token(token, line_lengths) yield ColorSpan(span, "keyword") diff --git a/Misc/NEWS.d/next/Library/2025-06-01-11-14-00.gh-issue-134953.ashdfs.rst b/Misc/NEWS.d/next/Library/2025-06-01-11-14-00.gh-issue-134953.ashdfs.rst new file mode 100644 index 00000000000000..c2f112dc62cea8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-01-11-14-00.gh-issue-134953.ashdfs.rst @@ -0,0 +1,2 @@ +Expand ``_colorize`` theme with ``keyword_constant`` and implement in +:term:`repl`. From a003112821a445128f9b94f9a46528e5449dfc86 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 15 Sep 2025 17:36:32 +0300 Subject: [PATCH 07/18] gh-138712: Add os.NODEV (GH-138728) --- Doc/library/os.rst | 7 ++++++ Lib/test/test_posix.py | 22 +++++++++++++++++-- ...-09-10-10-11-59.gh-issue-138712.avrPG5.rst | 1 + Modules/posixmodule.c | 4 ++++ 4 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-09-10-10-11-59.gh-issue-138712.avrPG5.rst diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 0333fe9f9967f8..2e04fbb6f63fd3 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -2630,6 +2630,13 @@ features: Compose a raw device number from the major and minor device numbers. +.. data:: NODEV + + Non-existent device. + + .. versionadded:: next + + .. function:: pathconf(path, name) Return system configuration information relevant to a named file. *name* diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 0bb65fe717d359..ab3d128d08ab47 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -757,12 +757,30 @@ def test_makedev(self): self.assertRaises((ValueError, OverflowError), posix.makedev, x, minor) self.assertRaises((ValueError, OverflowError), posix.makedev, major, x) - if sys.platform == 'linux' and not support.linked_to_musl(): - NODEV = -1 + # The following tests are needed to test functions accepting or + # returning the special value NODEV (if it is defined). major(), minor() + # and makefile() are the only easily reproducible examples, but that + # behavior is platform specific -- on some platforms their code has + # a special case for NODEV, on others this is just an implementation + # artifact. + if (hasattr(posix, 'NODEV') and + sys.platform.startswith(('linux', 'macos', 'freebsd', 'dragonfly', + 'sunos'))): + NODEV = posix.NODEV self.assertEqual(posix.major(NODEV), NODEV) self.assertEqual(posix.minor(NODEV), NODEV) self.assertEqual(posix.makedev(NODEV, NODEV), NODEV) + def test_nodev(self): + # NODEV is not a part of Posix, but is defined on many systems. + if (not hasattr(posix, 'NODEV') + and (not sys.platform.startswith(('linux', 'macos', 'freebsd', + 'dragonfly', 'netbsd', 'openbsd', + 'sunos')) + or support.linked_to_musl())): + self.skipTest('not defined on this platform') + self.assertHasAttr(posix, 'NODEV') + def _test_all_chown_common(self, chown_func, first_param, stat_func): """Common code for chown, fchown and lchown tests.""" def check_stat(uid, gid): diff --git a/Misc/NEWS.d/next/Library/2025-09-10-10-11-59.gh-issue-138712.avrPG5.rst b/Misc/NEWS.d/next/Library/2025-09-10-10-11-59.gh-issue-138712.avrPG5.rst new file mode 100644 index 00000000000000..3917726bd048dc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-09-10-10-11-59.gh-issue-138712.avrPG5.rst @@ -0,0 +1 @@ +Add :const:`os.NODEV`. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index bba73c659dd168..62b0c35602323f 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -17861,6 +17861,10 @@ all_ins(PyObject *m) #endif #endif /* HAVE_EVENTFD && EFD_CLOEXEC */ +#ifdef NODEV + if (PyModule_Add(m, "NODEV", _PyLong_FromDev(NODEV))) return -1; +#endif + #if defined(__APPLE__) if (PyModule_AddIntConstant(m, "_COPYFILE_DATA", COPYFILE_DATA)) return -1; if (PyModule_AddIntConstant(m, "_COPYFILE_STAT", COPYFILE_STAT)) return -1; From 5c4bb9b7f6a779351afcdd76f390c572b3c1dc06 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Mon, 15 Sep 2025 20:09:15 +0530 Subject: [PATCH 08/18] gh-137992: fix `PyRefTracer_SetTracer` to start world before returning (#138925) fix deadlock in PyRefTracer_SetTracer --- Objects/object.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Objects/object.c b/Objects/object.c index c9bcc0c7b09e63..1f10c2531fead1 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -3292,6 +3292,7 @@ int PyRefTracer_SetTracer(PyRefTracer tracer, void *data) { if (_PyRuntime.ref_tracer.tracer_func != NULL) { _PyReftracerTrack(NULL, PyRefTracer_TRACKER_REMOVED); if (PyErr_Occurred()) { + _PyEval_StartTheWorldAll(&_PyRuntime); return -1; } } From 9c9a0f7da7bf626b6d156c9fe3df22597ee3fe9e Mon Sep 17 00:00:00 2001 From: Savannah Bailey Date: Mon, 15 Sep 2025 16:29:45 +0100 Subject: [PATCH 09/18] GH-132732: Use pure op machinery to optimize various instructions with `_POP_TOP` and `_POP_TWO` (#137577) --- Lib/test/test_capi/test_opt.py | 85 ++++++++++ ...-08-09-04-07-05.gh-issue-132732.8BiIVJ.rst | 1 + Python/optimizer_bytecodes.c | 9 ++ Python/optimizer_cases.c.h | 151 ++++++++++++++++-- Tools/cases_generator/optimizer_generator.py | 36 +++-- 5 files changed, 259 insertions(+), 23 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-08-09-04-07-05.gh-issue-132732.8BiIVJ.rst diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index ffd65dbb1464f8..9601cedfe56f48 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1614,6 +1614,74 @@ def f(n): # But all of the appends we care about are still there: self.assertEqual(uops.count("_CALL_LIST_APPEND"), len("ABCDEFG")) + def test_unary_negative_pop_top_load_const_inline_borrow(self): + def testfunc(n): + x = 0 + for i in range(n): + a = 1 + result = -a + if result < 0: + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_UNARY_NEGATIVE", uops) + self.assertNotIn("_POP_TOP_LOAD_CONST_INLINE_BORROW", uops) + + def test_unary_not_pop_top_load_const_inline_borrow(self): + def testfunc(n): + x = 0 + for i in range(n): + a = 42 + result = not a + if result: + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, 0) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_UNARY_NOT", uops) + self.assertNotIn("_POP_TOP_LOAD_CONST_INLINE_BORROW", uops) + + def test_unary_invert_pop_top_load_const_inline_borrow(self): + def testfunc(n): + x = 0 + for i in range(n): + a = 0 + result = ~a + if result < 0: + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_UNARY_INVERT", uops) + self.assertNotIn("_POP_TOP_LOAD_CONST_INLINE_BORROW", uops) + + def test_compare_op_pop_two_load_const_inline_borrow(self): + def testfunc(n): + x = 0 + for _ in range(n): + a = 10 + b = 10.0 + if a == b: + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_COMPARE_OP", uops) + self.assertNotIn("_POP_TWO_LOAD_CONST_INLINE_BORROW", uops) + def test_compare_op_int_pop_two_load_const_inline_borrow(self): def testfunc(n): x = 0 @@ -1665,6 +1733,23 @@ def testfunc(n): self.assertNotIn("_COMPARE_OP_FLOAT", uops) self.assertNotIn("_POP_TWO_LOAD_CONST_INLINE_BORROW", uops) + def test_contains_op_pop_two_load_const_inline_borrow(self): + def testfunc(n): + x = 0 + for _ in range(n): + a = "foo" + s = "foo bar baz" + if a in s: + x += 1 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertNotIn("_CONTAINS_OP", uops) + self.assertNotIn("_POP_TWO_LOAD_CONST_INLINE_BORROW", uops) + def test_to_bool_bool_contains_op_set(self): """ Test that _TO_BOOL_BOOL is removed from code like: diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-08-09-04-07-05.gh-issue-132732.8BiIVJ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-09-04-07-05.gh-issue-132732.8BiIVJ.rst new file mode 100644 index 00000000000000..c1fa14e0566e15 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-09-04-07-05.gh-issue-132732.8BiIVJ.rst @@ -0,0 +1 @@ +Optimize ``_COMPARE_OP``, ``_CONTAINS_OP``, ``_UNARY_NEGATIVE``, ``_UNARY_NOT``, and ``_UNARY_INVERT`` in JIT builds with constant-loading uops (``_POP_TWO_LOAD_CONST_INLINE_BORROW`` and ``_POP_TOP_LOAD_CONST_INLINE_BORROW``), and then remove both to reduce instruction count. diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index eccbddf0546ab3..8f719f5750bd91 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -397,6 +397,7 @@ dummy_func(void) { } op(_UNARY_NEGATIVE, (value -- res)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(value); if (sym_is_compact_int(value)) { res = sym_new_compact_int(ctx); } @@ -412,6 +413,10 @@ dummy_func(void) { } op(_UNARY_INVERT, (value -- res)) { + // Required to avoid a warning due to the deprecation of bitwise inversion of bools + if (!sym_matches_type(value, &PyBool_Type)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(value); + } if (sym_matches_type(value, &PyLong_Type)) { res = sym_new_type(ctx, &PyLong_Type); } @@ -421,6 +426,9 @@ dummy_func(void) { } op(_COMPARE_OP, (left, right -- res)) { + // Comparison between bytes and str or int is not impacted by this optimization as bytes + // is not a safe type (due to its ability to raise a warning during comparisons). + REPLACE_OPCODE_IF_EVALUATES_PURE(left, right); if (oparg & 16) { res = sym_new_type(ctx, &PyBool_Type); } @@ -449,6 +457,7 @@ dummy_func(void) { } op(_CONTAINS_OP, (left, right -- b)) { + REPLACE_OPCODE_IF_EVALUATES_PURE(left, right); b = sym_new_type(ctx, &PyBool_Type); } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 8617355e25f418..99601b016acc15 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -188,6 +188,31 @@ JitOptRef value; JitOptRef res; value = stack_pointer[-1]; + if ( + sym_is_safe_const(ctx, value) + ) { + JitOptRef value_sym = value; + _PyStackRef value = sym_get_const_as_stackref(ctx, value_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); + PyStackRef_CLOSE(value); + if (res_o == NULL) { + goto error; + } + res_stackref = PyStackRef_FromPyObjectSteal(res_o); + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } + stack_pointer[-1] = res; + break; + } if (sym_is_compact_int(value)) { res = sym_new_compact_int(ctx); } @@ -220,6 +245,13 @@ ? PyStackRef_True : PyStackRef_False; /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } stack_pointer[-1] = res; break; } @@ -359,6 +391,33 @@ JitOptRef value; JitOptRef res; value = stack_pointer[-1]; + if (!sym_matches_type(value, &PyBool_Type)) { + if ( + sym_is_safe_const(ctx, value) + ) { + JitOptRef value_sym = value; + _PyStackRef value = sym_get_const_as_stackref(ctx, value_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); + PyStackRef_CLOSE(value); + if (res_o == NULL) { + goto error; + } + res_stackref = PyStackRef_FromPyObjectSteal(res_o); + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TOP_LOAD_CONST_INLINE_BORROW since we have one input and an immortal result + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } + stack_pointer[-1] = res; + break; + } + } if (sym_matches_type(value, &PyLong_Type)) { res = sym_new_type(ctx, &PyLong_Type); } @@ -438,7 +497,6 @@ PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { @@ -489,7 +547,6 @@ PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { @@ -540,7 +597,6 @@ PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { @@ -610,7 +666,6 @@ } /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { @@ -663,7 +718,6 @@ } /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { @@ -716,7 +770,6 @@ } /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { @@ -796,7 +849,6 @@ res_stackref = PyStackRef_FromPyObjectSteal(res_o); /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { @@ -1642,7 +1694,53 @@ } case _COMPARE_OP: { + JitOptRef right; + JitOptRef left; JitOptRef res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if ( + sym_is_safe_const(ctx, left) && + sym_is_safe_const(ctx, right) + ) { + JitOptRef left_sym = left; + JitOptRef right_sym = right; + _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); + _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); + _PyStackRef res_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert((oparg >> 5) <= Py_GE); + PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); + if (res_o == NULL) { + goto error; + } + if (oparg & 16) { + int res_bool = PyObject_IsTrue(res_o); + Py_DECREF(res_o); + if (res_bool < 0) { + goto error; + } + res_stackref = res_bool ? PyStackRef_True : PyStackRef_False; + } + else { + res_stackref = PyStackRef_FromPyObjectSteal(res_o); + } + /* End of uop copied from bytecodes for constant evaluation */ + res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); + if (sym_is_const(ctx, res)) { + PyObject *result = sym_get_const(ctx, res); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } if (oparg & 16) { res = sym_new_type(ctx, &PyBool_Type); } @@ -1682,7 +1780,6 @@ res_stackref = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { @@ -1733,7 +1830,6 @@ res_stackref = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { @@ -1782,7 +1878,6 @@ res_stackref = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { @@ -1812,7 +1907,42 @@ } case _CONTAINS_OP: { + JitOptRef right; + JitOptRef left; JitOptRef b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if ( + sym_is_safe_const(ctx, left) && + sym_is_safe_const(ctx, right) + ) { + JitOptRef left_sym = left; + JitOptRef right_sym = right; + _PyStackRef left = sym_get_const_as_stackref(ctx, left_sym); + _PyStackRef right = sym_get_const_as_stackref(ctx, right_sym); + _PyStackRef b_stackref; + /* Start of uop copied from bytecodes for constant evaluation */ + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + int res = PySequence_Contains(right_o, left_o); + if (res < 0) { + goto error; + } + b_stackref = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + /* End of uop copied from bytecodes for constant evaluation */ + b = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(b_stackref)); + if (sym_is_const(ctx, b)) { + PyObject *result = sym_get_const(ctx, b); + if (_Py_IsImmortal(result)) { + // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result + REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); + } + } + stack_pointer[-2] = b; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } b = sym_new_type(ctx, &PyBool_Type); stack_pointer[-2] = b; stack_pointer += -1; @@ -2885,7 +3015,6 @@ res_stackref = PyStackRef_FromPyObjectSteal(res_o); /* End of uop copied from bytecodes for constant evaluation */ res = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal(res_stackref)); - if (sym_is_const(ctx, res)) { PyObject *result = sym_get_const(ctx, res); if (_Py_IsImmortal(result)) { diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index b9985eaf48309d..7486fca245f5b9 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -4,6 +4,7 @@ """ import argparse +import textwrap from analyzer import ( Analysis, @@ -190,6 +191,7 @@ def replace_opcode_if_evaluates_pure( input_identifiers_as_str = {tkn.text for tkn in input_identifiers} used_stack_inputs = [inp for inp in uop.stack.inputs if inp.name in input_identifiers_as_str] assert len(used_stack_inputs) > 0 + self.out.start_line() emitter = OptimizerConstantEmitter(self.out, {}, self.original_uop, self.stack.copy()) emitter.emit("if (\n") for inp in used_stack_inputs[:-1]: @@ -232,18 +234,28 @@ def replace_opcode_if_evaluates_pure( emitter.emit(f"{outp.name} = sym_new_const_steal(ctx, PyStackRef_AsPyObjectSteal({outp.name}_stackref));\n") else: emitter.emit(f"{outp.name} = sym_new_const(ctx, PyStackRef_AsPyObjectBorrow({outp.name}_stackref));\n") - - if len(used_stack_inputs) == 2 and len(self.original_uop.stack.outputs) == 1: - outp = self.original_uop.stack.outputs[0] - if not outp.peek: - emitter.emit(f""" - if (sym_is_const(ctx, {outp.name})) {{ - PyObject *result = sym_get_const(ctx, {outp.name}); - if (_Py_IsImmortal(result)) {{ - // Replace with _POP_TWO_LOAD_CONST_INLINE_BORROW since we have two inputs and an immortal result - REPLACE_OP(this_instr, _POP_TWO_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)result); - }} - }}""") + if len(self.original_uop.stack.outputs) == 1: + outp = self.original_uop.stack.outputs[0] + if not outp.peek: + if self.original_uop.name.startswith('_'): + # Map input count to the appropriate constant-loading uop + input_count_to_uop = { + 1: "_POP_TOP_LOAD_CONST_INLINE_BORROW", + 2: "_POP_TWO_LOAD_CONST_INLINE_BORROW" + } + + input_count = len(used_stack_inputs) + if input_count in input_count_to_uop: + replacement_uop = input_count_to_uop[input_count] + input_desc = "one input" if input_count == 1 else "two inputs" + + emitter.emit(f"if (sym_is_const(ctx, {outp.name})) {{\n") + emitter.emit(f"PyObject *result = sym_get_const(ctx, {outp.name});\n") + emitter.emit(f"if (_Py_IsImmortal(result)) {{\n") + emitter.emit(f"// Replace with {replacement_uop} since we have {input_desc} and an immortal result\n") + emitter.emit(f"REPLACE_OP(this_instr, {replacement_uop}, 0, (uintptr_t)result);\n") + emitter.emit("}\n") + emitter.emit("}\n") storage.flush(self.out) emitter.emit("break;\n") From fa12c6bae47a41dd84e54b39d96bb73c4ba625c0 Mon Sep 17 00:00:00 2001 From: Savannah Bailey Date: Mon, 15 Sep 2025 17:09:51 +0100 Subject: [PATCH 10/18] GH-132732: Remove textwrap import (#138933) --- Tools/cases_generator/optimizer_generator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index 7486fca245f5b9..41df073cf6df23 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -4,7 +4,6 @@ """ import argparse -import textwrap from analyzer import ( Analysis, From 07d0b95b05dfaf5832f44c2fbc956761f9e29571 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 15 Sep 2025 19:20:31 +0300 Subject: [PATCH 11/18] gh-137490: Fix signal.sigwaitinfo() on NetBSD (GH-137523) Handle ECANCELED in the same way as EINTR to work around the Posix violation in the NetBSD's implementation. --- .../2025-08-07-17-18-57.gh-issue-137490.s89ieZ.rst | 2 ++ Modules/signalmodule.c | 8 +++++++- 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-08-07-17-18-57.gh-issue-137490.s89ieZ.rst diff --git a/Misc/NEWS.d/next/Library/2025-08-07-17-18-57.gh-issue-137490.s89ieZ.rst b/Misc/NEWS.d/next/Library/2025-08-07-17-18-57.gh-issue-137490.s89ieZ.rst new file mode 100644 index 00000000000000..bcb0938b8e3acb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-08-07-17-18-57.gh-issue-137490.s89ieZ.rst @@ -0,0 +1,2 @@ +Handle :data:`~errno.ECANCELED` in the same way as :data:`~errno.EINTR` in +:func:`signal.sigwaitinfo` on NetBSD. diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index 02bfdab957fc52..3c79ef1429087a 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -1183,7 +1183,13 @@ signal_sigwaitinfo_impl(PyObject *module, sigset_t sigset) err = sigwaitinfo(&sigset, &si); Py_END_ALLOW_THREADS } while (err == -1 - && errno == EINTR && !(async_err = PyErr_CheckSignals())); + && (errno == EINTR +#if defined(__NetBSD__) + /* NetBSD's implementation violates POSIX by setting + * errno to ECANCELED instead of EINTR. */ + || errno == ECANCELED +#endif + ) && !(async_err = PyErr_CheckSignals())); if (err == -1) return (!async_err) ? PyErr_SetFromErrno(PyExc_OSError) : NULL; From 26cfb1794255222b20cd7b502ab9193861df3184 Mon Sep 17 00:00:00 2001 From: 00ll00 <40747228+00ll00@users.noreply.github.com> Date: Tue, 16 Sep 2025 00:21:41 +0800 Subject: [PATCH 12/18] gh-138239: Fix incorrect highlighting of "type" in type statements in the REPL (GH-138241) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/_pyrepl/utils.py | 6 ++++++ Lib/test/test_pyrepl/test_reader.py | 8 +++++--- .../2025-08-29-12-56-55.gh-issue-138239.uthZFI.rst | 2 ++ 3 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-08-29-12-56-55.gh-issue-138239.uthZFI.rst diff --git a/Lib/_pyrepl/utils.py b/Lib/_pyrepl/utils.py index d32fce591fadcc..2ffd547f99d7c5 100644 --- a/Lib/_pyrepl/utils.py +++ b/Lib/_pyrepl/utils.py @@ -263,6 +263,12 @@ def is_soft_keyword_used(*tokens: TI | None) -> bool: return True case (TI(string="case"), TI(string="_"), TI(string=":")): return True + case ( + None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT) | TI(string=":"), + TI(string="type"), + TI(T.NAME, string=s) + ): + return not keyword.iskeyword(s) case _: return False diff --git a/Lib/test/test_pyrepl/test_reader.py b/Lib/test/test_pyrepl/test_reader.py index 9a02dff7387563..b1b6ae16a1e592 100644 --- a/Lib/test/test_pyrepl/test_reader.py +++ b/Lib/test/test_pyrepl/test_reader.py @@ -378,6 +378,7 @@ def funct(case: str = sys.platform) -> None: case "ios" | "android": print("on the phone") case _: print('arms around', match.group(1)) + type type = type[type] """ ) expected = dedent( @@ -397,6 +398,7 @@ def funct(case: str = sys.platform) -> None: {K}case{z} {s}"ios"{z} {o}|{z} {s}"android"{z}{o}:{z} {b}print{z}{o}({z}{s}"on the phone"{z}{o}){z} {K}case{z} {K}_{z}{o}:{z} {b}print{z}{o}({z}{s}'arms around'{z}{o},{z} match{o}.{z}group{o}({z}{n}1{z}{o}){z}{o}){z} + {K}type{z} {b}type{z} {o}={z} {b}type{z}{o}[{z}{b}type{z}{o}]{z} """ ) expected_sync = expected.format(a="", **colors) @@ -404,14 +406,14 @@ def funct(case: str = sys.platform) -> None: reader, _ = handle_all_events(events) self.assert_screen_equal(reader, code, clean=True) self.assert_screen_equal(reader, expected_sync) - self.assertEqual(reader.pos, 396) - self.assertEqual(reader.cxy, (0, 15)) + self.assertEqual(reader.pos, 419) + self.assertEqual(reader.cxy, (0, 16)) async_msg = "{k}async{z} ".format(**colors) expected_async = expected.format(a=async_msg, **colors) more_events = itertools.chain( code_to_events(code), - [Event(evt="key", data="up", raw=bytearray(b"\x1bOA"))] * 14, + [Event(evt="key", data="up", raw=bytearray(b"\x1bOA"))] * 15, code_to_events("async "), ) reader, _ = handle_all_events(more_events) diff --git a/Misc/NEWS.d/next/Library/2025-08-29-12-56-55.gh-issue-138239.uthZFI.rst b/Misc/NEWS.d/next/Library/2025-08-29-12-56-55.gh-issue-138239.uthZFI.rst new file mode 100644 index 00000000000000..9e0218e02664c6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-08-29-12-56-55.gh-issue-138239.uthZFI.rst @@ -0,0 +1,2 @@ +The REPL now highlights :keyword:`type` as a soft keyword +in :ref:`type statements `. From 46f823bb818b0e8f40b51c8fa9ef33f743915770 Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Mon, 15 Sep 2025 17:24:37 +0100 Subject: [PATCH 13/18] gh-132732: Clear errors in JIT optimizer on error (GH-136048) --- Python/optimizer_analysis.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index fd395d3c6c254f..9d43f2de41df78 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -462,7 +462,7 @@ const uint16_t op_without_decref_inputs[MAX_UOP_ID + 1] = { [_BINARY_OP_SUBTRACT_FLOAT] = _BINARY_OP_SUBTRACT_FLOAT__NO_DECREF_INPUTS, }; -/* 1 for success, 0 for not ready, cannot error at the moment. */ +/* >0 (length) for success, 0 for not ready, clears all possible errors. */ static int optimize_uops( PyCodeObject *co, @@ -472,6 +472,7 @@ optimize_uops( _PyBloomFilter *dependencies ) { + assert(!PyErr_Occurred()); JitOptContext context; JitOptContext *ctx = &context; @@ -555,7 +556,11 @@ optimize_uops( OPT_ERROR_IN_OPCODE(opcode); } _Py_uop_abstractcontext_fini(ctx); - return -1; + + assert(PyErr_Occurred()); + PyErr_Clear(); + + return 0; } @@ -702,10 +707,12 @@ _Py_uop_analyze_and_optimize( _PyFrame_GetCode(frame), buffer, length, curr_stacklen, dependencies); - if (length <= 0) { + if (length == 0) { return length; } + assert(length > 0); + length = remove_unneeded_uops(buffer, length); assert(length > 0); From 8ef7735c536e0ffe4a60224e59b7587288f53e9e Mon Sep 17 00:00:00 2001 From: yihong Date: Tue, 16 Sep 2025 00:26:23 +0800 Subject: [PATCH 14/18] gh-128636: Fix crash in PyREPL when os.environ is overwritten with an invalid value for macOS (GH-138089) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: yihong0618 Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/_colorize.py | 21 +++++++++++++------ Lib/_pyrepl/unix_console.py | 8 +++++-- Lib/test/support/__init__.py | 2 +- Lib/test/test_pyrepl/test_unix_console.py | 9 ++++++++ ...-09-10-10-02-59.gh-issue-128636.ldRKGZ.rst | 2 ++ 5 files changed, 33 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-09-10-10-02-59.gh-issue-128636.ldRKGZ.rst diff --git a/Lib/_colorize.py b/Lib/_colorize.py index f45e7b8bb300f1..d35486296f2684 100644 --- a/Lib/_colorize.py +++ b/Lib/_colorize.py @@ -288,21 +288,29 @@ def decolor(text: str) -> str: def can_colorize(*, file: IO[str] | IO[bytes] | None = None) -> bool: + + def _safe_getenv(k: str, fallback: str | None = None) -> str | None: + """Exception-safe environment retrieval. See gh-128636.""" + try: + return os.environ.get(k, fallback) + except Exception: + return fallback + if file is None: file = sys.stdout if not sys.flags.ignore_environment: - if os.environ.get("PYTHON_COLORS") == "0": + if _safe_getenv("PYTHON_COLORS") == "0": return False - if os.environ.get("PYTHON_COLORS") == "1": + if _safe_getenv("PYTHON_COLORS") == "1": return True - if os.environ.get("NO_COLOR"): + if _safe_getenv("NO_COLOR"): return False if not COLORIZE: return False - if os.environ.get("FORCE_COLOR"): + if _safe_getenv("FORCE_COLOR"): return True - if os.environ.get("TERM") == "dumb": + if _safe_getenv("TERM") == "dumb": return False if not hasattr(file, "fileno"): @@ -345,7 +353,8 @@ def get_theme( environment (including environment variable state and console configuration on Windows) can also change in the course of the application life cycle. """ - if force_color or (not force_no_color and can_colorize(file=tty_file)): + if force_color or (not force_no_color and + can_colorize(file=tty_file)): return _theme return theme_no_color diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index a7e49923191c07..9953051bf7c4ef 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -159,6 +159,10 @@ def __init__( self.pollob.register(self.input_fd, select.POLLIN) self.terminfo = terminfo.TermInfo(term or None) self.term = term + self.is_apple_terminal = ( + platform.system() == "Darwin" + and os.getenv("TERM_PROGRAM") == "Apple_Terminal" + ) @overload def _my_getstr(cap: str, optional: Literal[False] = False) -> bytes: ... @@ -339,7 +343,7 @@ def prepare(self): tcsetattr(self.input_fd, termios.TCSADRAIN, raw) # In macOS terminal we need to deactivate line wrap via ANSI escape code - if platform.system() == "Darwin" and os.getenv("TERM_PROGRAM") == "Apple_Terminal": + if self.is_apple_terminal: os.write(self.output_fd, b"\033[?7l") self.screen = [] @@ -370,7 +374,7 @@ def restore(self): self.flushoutput() tcsetattr(self.input_fd, termios.TCSADRAIN, self.__svtermstate) - if platform.system() == "Darwin" and os.getenv("TERM_PROGRAM") == "Apple_Terminal": + if self.is_apple_terminal: os.write(self.output_fd, b"\033[?7h") if hasattr(self, "old_sigwinch"): diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 4bfd01ed14a0a1..8d614ab3d42b5a 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2898,7 +2898,7 @@ def force_color(color: bool): from .os_helper import EnvironmentVarGuard with ( - swap_attr(_colorize, "can_colorize", lambda file=None: color), + swap_attr(_colorize, "can_colorize", lambda *, file=None: color), EnvironmentVarGuard() as env, ): env.unset("FORCE_COLOR", "NO_COLOR", "PYTHON_COLORS") diff --git a/Lib/test/test_pyrepl/test_unix_console.py b/Lib/test/test_pyrepl/test_unix_console.py index ab1236768cfb3e..6185c7e3c794e3 100644 --- a/Lib/test/test_pyrepl/test_unix_console.py +++ b/Lib/test/test_pyrepl/test_unix_console.py @@ -303,3 +303,12 @@ def test_getheightwidth_with_invalid_environ(self, _os_write): self.assertIsInstance(console.getheightwidth(), tuple) os.environ = [] self.assertIsInstance(console.getheightwidth(), tuple) + + @unittest.skipUnless(sys.platform == "darwin", "requires macOS") + def test_restore_with_invalid_environ_on_macos(self, _os_write): + # gh-128636 for macOS + console = UnixConsole(term="xterm") + with os_helper.EnvironmentVarGuard(): + os.environ = [] + console.prepare() # needed to call restore() + console.restore() # this should succeed diff --git a/Misc/NEWS.d/next/Library/2025-09-10-10-02-59.gh-issue-128636.ldRKGZ.rst b/Misc/NEWS.d/next/Library/2025-09-10-10-02-59.gh-issue-128636.ldRKGZ.rst new file mode 100644 index 00000000000000..54eae0a4601617 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-09-10-10-02-59.gh-issue-128636.ldRKGZ.rst @@ -0,0 +1,2 @@ +Fix crash in PyREPL when os.environ is overwritten with an invalid value for +mac From 811acc85d5b001e0bef6ac2e6b499e7c4f149262 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Mon, 15 Sep 2025 17:27:37 +0100 Subject: [PATCH 15/18] gh-134953: Make the True/False/None check more efficient (GH-138931) --- Lib/_pyrepl/utils.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Lib/_pyrepl/utils.py b/Lib/_pyrepl/utils.py index 2ffd547f99d7c5..64708e843b685b 100644 --- a/Lib/_pyrepl/utils.py +++ b/Lib/_pyrepl/utils.py @@ -20,6 +20,7 @@ ZERO_WIDTH_BRACKET = re.compile(r"\x01.*?\x02") ZERO_WIDTH_TRANS = str.maketrans({"\x01": "", "\x02": ""}) IDENTIFIERS_AFTER = {"def", "class"} +KEYWORD_CONSTANTS = {"True", "False", "None"} BUILTINS = {str(name) for name in dir(builtins) if not name.startswith('_')} @@ -196,12 +197,12 @@ def gen_colors_from_token_stream( is_def_name = False span = Span.from_token(token, line_lengths) yield ColorSpan(span, "definition") - elif token.string in ("True", "False", "None"): - span = Span.from_token(token, line_lengths) - yield ColorSpan(span, "keyword_constant") elif keyword.iskeyword(token.string): + span_cls = "keyword" + if token.string in KEYWORD_CONSTANTS: + span_cls = "keyword_constant" span = Span.from_token(token, line_lengths) - yield ColorSpan(span, "keyword") + yield ColorSpan(span, span_cls) if token.string in IDENTIFIERS_AFTER: is_def_name = True elif ( From 29d026f93e14cc9bf5c17ac195ab0ec7708eaf57 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 15 Sep 2025 19:40:28 +0300 Subject: [PATCH 16/18] gh-37817: Allow assignment to __bases__ of direct subclasses of builtin classes (GH-137585) --- Lib/test/test_descr.py | 205 ++++++++++++++---- ...5-08-09-11-38-37.gh-issue-37817.Y5Fhde.rst | 2 + Objects/typeobject.c | 61 ++++-- 3 files changed, 202 insertions(+), 66 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-08-09-11-38-37.gh-issue-37817.Y5Fhde.rst diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index 9dfeeccb81b34d..39b835b03fc599 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -4077,42 +4077,167 @@ class E(D): self.assertEqual(e.a, 2) self.assertEqual(C2.__subclasses__(), [D]) - try: + with self.assertRaisesRegex(TypeError, + "cannot delete '__bases__' attribute of immutable type"): del D.__bases__ - except (TypeError, AttributeError): - pass - else: - self.fail("shouldn't be able to delete .__bases__") - - try: + with self.assertRaisesRegex(TypeError, 'can only assign non-empty tuple'): D.__bases__ = () - except TypeError as msg: - if str(msg) == "a new-style class can't have only classic bases": - self.fail("wrong error message for .__bases__ = ()") - else: - self.fail("shouldn't be able to set .__bases__ to ()") - - try: + with self.assertRaisesRegex(TypeError, 'can only assign tuple'): + D.__bases__ = [C] + with self.assertRaisesRegex(TypeError, 'duplicate base class'): + D.__bases__ = (C, C) + with self.assertRaisesRegex(TypeError, 'inheritance cycle'): D.__bases__ = (D,) - except TypeError: - pass - else: - # actually, we'll have crashed by here... - self.fail("shouldn't be able to create inheritance cycles") + with self.assertRaisesRegex(TypeError, 'inheritance cycle'): + D.__bases__ = (E,) - try: - D.__bases__ = (C, C) - except TypeError: - pass - else: - self.fail("didn't detect repeated base classes") + class A: + __slots__ = () + def __repr__(self): + return '' + class A_with_dict: + __slots__ = ('__dict__',) + def __repr__(self): + return '' + class A_with_dict_weakref: + def __repr__(self): + return '' + class A_with_slots: + __slots__ = ('x',) + def __repr__(self): + return '' + class A_with_slots_dict: + __slots__ = ('x', '__dict__') + def __repr__(self): + return '' - try: - D.__bases__ = (E,) - except TypeError: - pass - else: - self.fail("shouldn't be able to create inheritance cycles") + class B: + __slots__ = () + b = B() + r = repr(b) + with self.assertRaisesRegex(TypeError, 'layout differs'): + B.__bases__ = (int,) + with self.assertRaisesRegex(TypeError, 'layout differs'): + B.__bases__ = (A_with_dict_weakref,) + with self.assertRaisesRegex(TypeError, 'layout differs'): + B.__bases__ = (A_with_dict,) + with self.assertRaisesRegex(TypeError, 'layout differs'): + B.__bases__ = (A_with_slots,) + B.__bases__ = (A,) + self.assertNotHasAttr(b, '__dict__') + self.assertNotHasAttr(b, '__weakref__') + self.assertEqual(repr(b), '') + B.__bases__ = (object,) + self.assertEqual(repr(b), r) + + class B_with_dict_weakref: + pass + b = B_with_dict_weakref() + with self.assertRaisesRegex(TypeError, 'layout differs'): + B.__bases__ = (A_with_slots,) + B_with_dict_weakref.__bases__ = (A_with_dict_weakref,) + self.assertEqual(repr(b), '') + B_with_dict_weakref.__bases__ = (A_with_dict,) + self.assertEqual(repr(b), '') + B_with_dict_weakref.__bases__ = (A,) + self.assertEqual(repr(b), '') + B_with_dict_weakref.__bases__ = (object,) + + class B_with_slots: + __slots__ = ('x',) + b = B_with_slots() + with self.assertRaisesRegex(TypeError, 'layout differs'): + B_with_slots.__bases__ = (A_with_dict_weakref,) + with self.assertRaisesRegex(TypeError, 'layout differs'): + B_with_slots.__bases__ = (A_with_dict,) + B_with_slots.__bases__ = (A,) + self.assertEqual(repr(b), '') + + class B_with_slots_dict: + __slots__ = ('x', '__dict__') + b = B_with_slots_dict() + with self.assertRaisesRegex(TypeError, 'layout differs'): + B_with_slots_dict.__bases__ = (A_with_dict_weakref,) + B_with_slots_dict.__bases__ = (A_with_dict,) + self.assertEqual(repr(b), '') + B_with_slots_dict.__bases__ = (A,) + self.assertEqual(repr(b), '') + + class B_with_slots_dict_weakref: + __slots__ = ('x', '__dict__', '__weakref__') + b = B_with_slots_dict_weakref() + with self.assertRaisesRegex(TypeError, 'layout differs'): + B_with_slots_dict_weakref.__bases__ = (A_with_slots_dict,) + with self.assertRaisesRegex(TypeError, 'layout differs'): + B_with_slots_dict_weakref.__bases__ = (A_with_slots,) + B_with_slots_dict_weakref.__bases__ = (A_with_dict_weakref,) + self.assertEqual(repr(b), '') + B_with_slots_dict_weakref.__bases__ = (A_with_dict,) + self.assertEqual(repr(b), '') + B_with_slots_dict_weakref.__bases__ = (A,) + self.assertEqual(repr(b), '') + + class C_with_slots(A_with_slots): + __slots__ = () + c = C_with_slots() + with self.assertRaisesRegex(TypeError, 'layout differs'): + C_with_slots.__bases__ = (A_with_slots_dict,) + with self.assertRaisesRegex(TypeError, 'layout differs'): + C_with_slots.__bases__ = (A_with_dict_weakref,) + with self.assertRaisesRegex(TypeError, 'layout differs'): + C_with_slots.__bases__ = (A_with_dict,) + with self.assertRaisesRegex(TypeError, 'layout differs'): + C_with_slots.__bases__ = (A,) + C_with_slots.__bases__ = (A_with_slots,) + self.assertEqual(repr(c), '') + + class C_with_slots_dict(A_with_slots): + pass + c = C_with_slots_dict() + with self.assertRaisesRegex(TypeError, 'layout differs'): + C_with_slots_dict.__bases__ = (A_with_dict_weakref,) + with self.assertRaisesRegex(TypeError, 'layout differs'): + C_with_slots_dict.__bases__ = (A_with_dict,) + with self.assertRaisesRegex(TypeError, 'layout differs'): + C_with_slots_dict.__bases__ = (A,) + C_with_slots_dict.__bases__ = (A_with_slots_dict,) + self.assertEqual(repr(c), '') + C_with_slots_dict.__bases__ = (A_with_slots,) + self.assertEqual(repr(c), '') + + class A_int(int): + __slots__ = () + def __repr__(self): + return '' + class B_int(int): + __slots__ = () + b = B_int(42) + with self.assertRaisesRegex(TypeError, 'layout differs'): + B_int.__bases__ = (object,) + with self.assertRaisesRegex(TypeError, 'layout differs'): + B_int.__bases__ = (tuple,) + with self.assertRaisesRegex(TypeError, 'is not an acceptable base type'): + B_int.__bases__ = (bool,) + B_int.__bases__ = (A_int,) + self.assertEqual(repr(b), '') + B_int.__bases__ = (int,) + self.assertEqual(repr(b), '42') + + class A_tuple(tuple): + __slots__ = () + def __repr__(self): + return '' + class B_tuple(tuple): + __slots__ = () + b = B_tuple((1, 2)) + with self.assertRaisesRegex(TypeError, 'layout differs'): + B_tuple.__bases__ = (object,) + with self.assertRaisesRegex(TypeError, 'layout differs'): + B_tuple.__bases__ = (int,) + B_tuple.__bases__ = (A_tuple,) + self.assertEqual(repr(b), '') + B_tuple.__bases__ = (tuple,) + self.assertEqual(repr(b), '(1, 2)') def test_assign_bases_many_subclasses(self): # This is intended to check that typeobject.c:queue_slot_update() can @@ -4165,26 +4290,14 @@ class C(object): class D(C): pass - try: + with self.assertRaisesRegex(TypeError, 'layout differs'): L.__bases__ = (dict,) - except TypeError: - pass - else: - self.fail("shouldn't turn list subclass into dict subclass") - try: + with self.assertRaisesRegex(TypeError, 'immutable type'): list.__bases__ = (dict,) - except TypeError: - pass - else: - self.fail("shouldn't be able to assign to list.__bases__") - try: + with self.assertRaisesRegex(TypeError, 'layout differs'): D.__bases__ = (C, list) - except TypeError: - pass - else: - self.fail("best_base calculation found wanting") def test_unsubclassable_types(self): with self.assertRaises(TypeError): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-08-09-11-38-37.gh-issue-37817.Y5Fhde.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-09-11-38-37.gh-issue-37817.Y5Fhde.rst new file mode 100644 index 00000000000000..5e73188ff2d694 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-09-11-38-37.gh-issue-37817.Y5Fhde.rst @@ -0,0 +1,2 @@ +Allow assignment to :attr:`~type.__bases__` of direct subclasses of builtin +classes. diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 9cead729b6fe7a..06f3ace1764a86 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1748,7 +1748,7 @@ type_get_mro(PyObject *tp, void *Py_UNUSED(closure)) static PyTypeObject *find_best_base(PyObject *); static int mro_internal(PyTypeObject *, int, PyObject **); static int type_is_subtype_base_chain(PyTypeObject *, PyTypeObject *); -static int compatible_for_assignment(PyTypeObject *, PyTypeObject *, const char *); +static int compatible_for_assignment(PyTypeObject *, PyTypeObject *, const char *, int); static int add_subclass(PyTypeObject*, PyTypeObject*); static int add_all_subclasses(PyTypeObject *type, PyObject *bases); static void remove_subclass(PyTypeObject *, PyTypeObject *); @@ -1886,7 +1886,7 @@ type_check_new_bases(PyTypeObject *type, PyObject *new_bases, PyTypeObject **bes if (*best_base == NULL) return -1; - if (!compatible_for_assignment(type->tp_base, *best_base, "__bases__")) { + if (!compatible_for_assignment(type, *best_base, "__bases__", 0)) { return -1; } @@ -7263,10 +7263,6 @@ compatible_with_tp_base(PyTypeObject *child) return (parent != NULL && child->tp_basicsize == parent->tp_basicsize && child->tp_itemsize == parent->tp_itemsize && - child->tp_dictoffset == parent->tp_dictoffset && - child->tp_weaklistoffset == parent->tp_weaklistoffset && - ((child->tp_flags & Py_TPFLAGS_HAVE_GC) == - (parent->tp_flags & Py_TPFLAGS_HAVE_GC)) && (child->tp_dealloc == subtype_dealloc || child->tp_dealloc == parent->tp_dealloc)); } @@ -7301,11 +7297,24 @@ same_slots_added(PyTypeObject *a, PyTypeObject *b) } static int -compatible_for_assignment(PyTypeObject* oldto, PyTypeObject* newto, const char* attr) +compatible_flags(int setclass, PyTypeObject *origto, PyTypeObject *newto, unsigned long flags) +{ + /* For __class__ assignment, the flags should be the same. + For __bases__ assignment, the new base flags can only be set + if the original class flags are set. + */ + return setclass ? (origto->tp_flags & flags) == (newto->tp_flags & flags) + : !(~(origto->tp_flags & flags) & (newto->tp_flags & flags)); +} + +static int +compatible_for_assignment(PyTypeObject *origto, PyTypeObject *newto, + const char *attr, int setclass) { PyTypeObject *newbase, *oldbase; + PyTypeObject *oldto = setclass ? origto : origto->tp_base; - if (newto->tp_free != oldto->tp_free) { + if (setclass && newto->tp_free != oldto->tp_free) { PyErr_Format(PyExc_TypeError, "%s assignment: " "'%s' deallocator differs from '%s'", @@ -7314,6 +7323,28 @@ compatible_for_assignment(PyTypeObject* oldto, PyTypeObject* newto, const char* oldto->tp_name); return 0; } + if (!compatible_flags(setclass, origto, newto, + Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_INLINE_VALUES | + Py_TPFLAGS_PREHEADER)) + { + goto differs; + } + /* For __class__ assignment, tp_dictoffset and tp_weaklistoffset should + be the same for old and new types. + For __bases__ assignment, they can only be set in the new base + if they are set in the original class with the same value. + */ + if ((setclass || newto->tp_dictoffset) + && origto->tp_dictoffset != newto->tp_dictoffset) + { + goto differs; + } + if ((setclass || newto->tp_weaklistoffset) + && origto->tp_weaklistoffset != newto->tp_weaklistoffset) + { + goto differs; + } /* It's tricky to tell if two arbitrary types are sufficiently compatible as to be interchangeable; e.g., even if they have the same tp_basicsize, they @@ -7335,17 +7366,7 @@ compatible_for_assignment(PyTypeObject* oldto, PyTypeObject* newto, const char* !same_slots_added(newbase, oldbase))) { goto differs; } - if ((oldto->tp_flags & Py_TPFLAGS_INLINE_VALUES) != - ((newto->tp_flags & Py_TPFLAGS_INLINE_VALUES))) - { - goto differs; - } - /* The above does not check for the preheader */ - if ((oldto->tp_flags & Py_TPFLAGS_PREHEADER) == - ((newto->tp_flags & Py_TPFLAGS_PREHEADER))) - { - return 1; - } + return 1; differs: PyErr_Format(PyExc_TypeError, "%s assignment: " @@ -7422,7 +7443,7 @@ object_set_class_world_stopped(PyObject *self, PyTypeObject *newto) return -1; } - if (compatible_for_assignment(oldto, newto, "__class__")) { + if (compatible_for_assignment(oldto, newto, "__class__", 1)) { /* Changing the class will change the implicit dict keys, * so we must materialize the dictionary first. */ if (oldto->tp_flags & Py_TPFLAGS_INLINE_VALUES) { From a68efdf09cfd81c64973d346ed34057f20172543 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 15 Sep 2025 17:41:43 +0100 Subject: [PATCH 17/18] gh-129813, PEP 782: Use PyBytesWriter in _hashopenssl (#138922) Replace PyBytes_FromStringAndSize(NULL, size) with the new public PyBytesWriter API. --- Modules/_hashopenssl.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 628e6dc11668e0..19089b009f7911 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -1629,7 +1629,6 @@ pbkdf2_hmac_impl(PyObject *module, const char *hash_name, { _hashlibstate *state = get_hashlib_state(module); PyObject *key_obj = NULL; - char *key; long dklen; int retval; @@ -1682,24 +1681,24 @@ pbkdf2_hmac_impl(PyObject *module, const char *hash_name, goto end; } - key_obj = PyBytes_FromStringAndSize(NULL, dklen); - if (key_obj == NULL) { + PyBytesWriter *writer = PyBytesWriter_Create(dklen); + if (writer == NULL) { goto end; } - key = PyBytes_AS_STRING(key_obj); Py_BEGIN_ALLOW_THREADS retval = PKCS5_PBKDF2_HMAC((const char *)password->buf, (int)password->len, (const unsigned char *)salt->buf, (int)salt->len, iterations, digest, dklen, - (unsigned char *)key); + (unsigned char *)PyBytesWriter_GetData(writer)); Py_END_ALLOW_THREADS if (!retval) { - Py_CLEAR(key_obj); + PyBytesWriter_Discard(writer); notify_ssl_error_occurred_in(Py_STRINGIFY(PKCS5_PBKDF2_HMAC)); goto end; } + key_obj = PyBytesWriter_Finish(writer); end: if (digest != NULL) { @@ -1799,7 +1798,7 @@ _hashlib_scrypt_impl(PyObject *module, Py_buffer *password, Py_buffer *salt, (const char *)password->buf, (size_t)password->len, (const unsigned char *)salt->buf, (size_t)salt->len, (uint64_t)n, (uint64_t)r, (uint64_t)p, (uint64_t)maxmem, - PyBytesWriter_GetData(writer), (size_t)dklen + (unsigned char *)PyBytesWriter_GetData(writer), (size_t)dklen ); Py_END_ALLOW_THREADS From 537133d2b63611ce1c04aac4c283c932dee9985a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Simon?= Date: Mon, 15 Sep 2025 18:51:34 +0200 Subject: [PATCH 18/18] gh-69605: Hardcode some stdlib submodules in PyREPL module completion (os.path, collections.abc...) (GH-138268) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Łukasz Langa --- Lib/_pyrepl/_module_completer.py | 31 +++++++++- Lib/test/test_pyrepl/test_pyrepl.py | 62 +++++++++++++++---- ...5-08-30-17-15-05.gh-issue-69605.KjBk99.rst | 1 + 3 files changed, 80 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-08-30-17-15-05.gh-issue-69605.KjBk99.rst diff --git a/Lib/_pyrepl/_module_completer.py b/Lib/_pyrepl/_module_completer.py index 1e9462a42156d4..cf59e007f4df80 100644 --- a/Lib/_pyrepl/_module_completer.py +++ b/Lib/_pyrepl/_module_completer.py @@ -1,9 +1,12 @@ from __future__ import annotations +import importlib +import os import pkgutil import sys import token import tokenize +from importlib.machinery import FileFinder from io import StringIO from contextlib import contextmanager from dataclasses import dataclass @@ -16,6 +19,15 @@ from typing import Any, Iterable, Iterator, Mapping +HARDCODED_SUBMODULES = { + # Standard library submodules that are not detected by pkgutil.iter_modules + # but can be imported, so should be proposed in completion + "collections": ["abc"], + "os": ["path"], + "xml.parsers.expat": ["errors", "model"], +} + + def make_default_module_completer() -> ModuleCompleter: # Inside pyrepl, __package__ is set to None by default return ModuleCompleter(namespace={'__package__': None}) @@ -41,6 +53,7 @@ def __init__(self, namespace: Mapping[str, Any] | None = None) -> None: self.namespace = namespace or {} self._global_cache: list[pkgutil.ModuleInfo] = [] self._curr_sys_path: list[str] = sys.path[:] + self._stdlib_path = os.path.dirname(importlib.__path__[0]) def get_completions(self, line: str) -> list[str] | None: """Return the next possible import completions for 'line'.""" @@ -95,12 +108,26 @@ def _find_modules(self, path: str, prefix: str) -> list[str]: return [] modules: Iterable[pkgutil.ModuleInfo] = self.global_cache + is_stdlib_import: bool | None = None for segment in path.split('.'): modules = [mod_info for mod_info in modules if mod_info.ispkg and mod_info.name == segment] + if is_stdlib_import is None: + # Top-level import decide if we import from stdlib or not + is_stdlib_import = all( + self._is_stdlib_module(mod_info) for mod_info in modules + ) modules = self.iter_submodules(modules) - return [module.name for module in modules - if self.is_suggestion_match(module.name, prefix)] + + module_names = [module.name for module in modules] + if is_stdlib_import: + module_names.extend(HARDCODED_SUBMODULES.get(path, ())) + return [module_name for module_name in module_names + if self.is_suggestion_match(module_name, prefix)] + + def _is_stdlib_module(self, module_info: pkgutil.ModuleInfo) -> bool: + return (isinstance(module_info.module_finder, FileFinder) + and module_info.module_finder.path == self._stdlib_path) def is_suggestion_match(self, module_name: str, prefix: str) -> bool: if prefix: diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index 8e4450fdf99ecd..47d384a209e9ac 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -1,3 +1,4 @@ +import importlib import io import itertools import os @@ -26,9 +27,16 @@ code_to_events, ) from _pyrepl.console import Event -from _pyrepl._module_completer import ImportParser, ModuleCompleter -from _pyrepl.readline import (ReadlineAlikeReader, ReadlineConfig, - _ReadlineWrapper) +from _pyrepl._module_completer import ( + ImportParser, + ModuleCompleter, + HARDCODED_SUBMODULES, +) +from _pyrepl.readline import ( + ReadlineAlikeReader, + ReadlineConfig, + _ReadlineWrapper, +) from _pyrepl.readline import multiline_input as readline_multiline_input try: @@ -930,7 +938,6 @@ def test_func(self): class TestPyReplModuleCompleter(TestCase): def setUp(self): - import importlib # Make iter_modules() search only the standard library. # This makes the test more reliable in case there are # other user packages/scripts on PYTHONPATH which can @@ -1013,14 +1020,6 @@ def test_sub_module_private_completions(self): self.assertEqual(output, expected) def test_builtin_completion_top_level(self): - import importlib - # Make iter_modules() search only the standard library. - # This makes the test more reliable in case there are - # other user packages/scripts on PYTHONPATH which can - # intefere with the completions. - lib_path = os.path.dirname(importlib.__path__[0]) - sys.path = [lib_path] - cases = ( ("import bui\t\n", "import builtins"), ("from bui\t\n", "from builtins"), @@ -1076,6 +1075,32 @@ def test_no_fallback_on_regular_completion(self): output = reader.readline() self.assertEqual(output, expected) + def test_hardcoded_stdlib_submodules(self): + cases = ( + ("import collections.\t\n", "import collections.abc"), + ("from os import \t\n", "from os import path"), + ("import xml.parsers.expat.\t\te\t\n\n", "import xml.parsers.expat.errors"), + ("from xml.parsers.expat import \t\tm\t\n\n", "from xml.parsers.expat import model"), + ) + for code, expected in cases: + with self.subTest(code=code): + events = code_to_events(code) + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, expected) + + def test_hardcoded_stdlib_submodules_not_proposed_if_local_import(self): + with tempfile.TemporaryDirectory() as _dir: + dir = pathlib.Path(_dir) + (dir / "collections").mkdir() + (dir / "collections" / "__init__.py").touch() + (dir / "collections" / "foo.py").touch() + with patch.object(sys, "path", [dir, *sys.path]): + events = code_to_events("import collections.\t\n") + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, "import collections.foo") + def test_get_path_and_prefix(self): cases = ( ('', ('', '')), @@ -1204,6 +1229,19 @@ def test_parse_error(self): with self.subTest(code=code): self.assertEqual(actual, None) + +class TestHardcodedSubmodules(TestCase): + def test_hardcoded_stdlib_submodules_are_importable(self): + for parent_path, submodules in HARDCODED_SUBMODULES.items(): + for module_name in submodules: + path = f"{parent_path}.{module_name}" + with self.subTest(path=path): + # We can't use importlib.util.find_spec here, + # since some hardcoded submodules parents are + # not proper packages + importlib.import_module(path) + + class TestPasteEvent(TestCase): def prepare_reader(self, events): console = FakeConsole(events) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-08-30-17-15-05.gh-issue-69605.KjBk99.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-30-17-15-05.gh-issue-69605.KjBk99.rst new file mode 100644 index 00000000000000..d855470fc2b326 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-30-17-15-05.gh-issue-69605.KjBk99.rst @@ -0,0 +1 @@ +Fix some standard library submodules missing from the :term:`REPL` auto-completion of imports.