diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index badc97808c6132..9cdaa950e3479a 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -939,7 +939,7 @@ struct _is { struct _PyExecutorObject *executor_deletion_list_head; struct _PyExecutorObject *cold_executor; int executor_deletion_list_remaining_capacity; - size_t trace_run_counter; + size_t executor_creation_counter; _rare_events rare_events; PyDict_WatchCallback builtins_dict_watcher; diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 685c39dcd65fb9..8ed5436eb6838c 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -90,8 +90,9 @@ PyAPI_FUNC(void) _Py_Executors_InvalidateCold(PyInterpreterState *interp); #endif // Used as the threshold to trigger executor invalidation when -// trace_run_counter is greater than this value. -#define JIT_CLEANUP_THRESHOLD 100000 +// executor_creation_counter is greater than this value. +// This value is arbitrary and was not optimized. +#define JIT_CLEANUP_THRESHOLD 1000 #define TRACE_STACK_SIZE 5 diff --git a/Include/object.h b/Include/object.h index 9585f4a1d67a52..7f4b35df3b6263 100644 --- a/Include/object.h +++ b/Include/object.h @@ -695,8 +695,13 @@ PyAPI_DATA(PyObject) _Py_NotImplementedStruct; /* Don't use this directly */ # define Py_NotImplemented (&_Py_NotImplementedStruct) #endif -/* Macro for returning Py_NotImplemented from a function */ -#define Py_RETURN_NOTIMPLEMENTED return Py_NotImplemented +/* Macro for returning Py_NotImplemented from a function. Only treat + * Py_NotImplemented as immortal in the limited C API 3.12 and newer. */ +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 < 0x030c0000 +# define Py_RETURN_NOTIMPLEMENTED return Py_NewRef(Py_NotImplemented) +#else +# define Py_RETURN_NOTIMPLEMENTED return Py_NotImplemented +#endif /* Rich comparison opcodes */ #define Py_LT 0 diff --git a/Lib/test/test_free_threading/test_bz2.py b/Lib/test/test_free_threading/test_bz2.py new file mode 100644 index 00000000000000..0e09c64d5610a3 --- /dev/null +++ b/Lib/test/test_free_threading/test_bz2.py @@ -0,0 +1,53 @@ +import unittest + +from test.support import import_helper, threading_helper +from test.support.threading_helper import run_concurrently + +bz2 = import_helper.import_module("bz2") +from bz2 import BZ2Compressor, BZ2Decompressor + +from test.test_bz2 import ext_decompress, BaseTest + + +NTHREADS = 10 +TEXT = BaseTest.TEXT + + +@threading_helper.requires_working_threading() +class TestBZ2(unittest.TestCase): + def test_compressor(self): + bz2c = BZ2Compressor() + + def worker(): + # it should return empty bytes as it buffers data internally + data = bz2c.compress(TEXT) + self.assertEqual(data, b"") + + run_concurrently(worker_func=worker, nthreads=NTHREADS) + data = bz2c.flush() + # The decompressed data should be TEXT repeated NTHREADS times + decompressed = ext_decompress(data) + self.assertEqual(decompressed, TEXT * NTHREADS) + + def test_decompressor(self): + chunk_size = 128 + chunks = [bytes([ord("a") + i]) * chunk_size for i in range(NTHREADS)] + input_data = b"".join(chunks) + compressed = bz2.compress(input_data) + + bz2d = BZ2Decompressor() + output = [] + + def worker(): + data = bz2d.decompress(compressed, chunk_size) + self.assertEqual(len(data), chunk_size) + output.append(data) + + run_concurrently(worker_func=worker, nthreads=NTHREADS) + self.assertEqual(len(output), NTHREADS) + # Verify the expected chunks (order doesn't matter due to append race) + self.assertEqual(set(output), set(chunks)) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index ebc25a589876a0..c7e81fff6f776b 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -759,7 +759,6 @@ class TestStrftime4dyear(_TestStrftimeYear, _Test4dYear, unittest.TestCase): class TestPytime(unittest.TestCase): @skip_if_buggy_ucrt_strfptime - @unittest.skipUnless(time._STRUCT_TM_ITEMS == 11, "needs tm_zone support") def test_localtime_timezone(self): # Get the localtime and examine it for the offset and zone. @@ -794,14 +793,12 @@ def test_localtime_timezone(self): self.assertEqual(new_lt.tm_gmtoff, lt.tm_gmtoff) self.assertEqual(new_lt9.tm_zone, lt.tm_zone) - @unittest.skipUnless(time._STRUCT_TM_ITEMS == 11, "needs tm_zone support") def test_strptime_timezone(self): t = time.strptime("UTC", "%Z") self.assertEqual(t.tm_zone, 'UTC') t = time.strptime("+0500", "%z") self.assertEqual(t.tm_gmtoff, 5 * 3600) - @unittest.skipUnless(time._STRUCT_TM_ITEMS == 11, "needs tm_zone support") def test_short_times(self): import pickle diff --git a/Misc/NEWS.d/next/C_API/2025-10-26-16-45-06.gh-issue-140487.fGOqss.rst b/Misc/NEWS.d/next/C_API/2025-10-26-16-45-06.gh-issue-140487.fGOqss.rst new file mode 100644 index 00000000000000..16b0d9d4084ba0 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-10-26-16-45-06.gh-issue-140487.fGOqss.rst @@ -0,0 +1,2 @@ +Fix :c:macro:`Py_RETURN_NOTIMPLEMENTED` in limited C API 3.11 and older: +don't treat ``Py_NotImplemented`` as immortal. Patch by Victor Stinner. diff --git a/Modules/_bz2module.c b/Modules/_bz2module.c index 9721b493a19956..452b88dfed29ce 100644 --- a/Modules/_bz2module.c +++ b/Modules/_bz2module.c @@ -97,20 +97,11 @@ OutputBuffer_OnError(_BlocksOutputBuffer *buffer) #endif /* ! BZ_CONFIG_ERROR */ -#define ACQUIRE_LOCK(obj) do { \ - if (!PyThread_acquire_lock((obj)->lock, 0)) { \ - Py_BEGIN_ALLOW_THREADS \ - PyThread_acquire_lock((obj)->lock, 1); \ - Py_END_ALLOW_THREADS \ - } } while (0) -#define RELEASE_LOCK(obj) PyThread_release_lock((obj)->lock) - - typedef struct { PyObject_HEAD bz_stream bzs; int flushed; - PyThread_type_lock lock; + PyMutex mutex; } BZ2Compressor; typedef struct { @@ -126,7 +117,7 @@ typedef struct { separately. Conversion and looping is encapsulated in decompress_buf() */ size_t bzs_avail_in_real; - PyThread_type_lock lock; + PyMutex mutex; } BZ2Decompressor; #define _BZ2Compressor_CAST(op) ((BZ2Compressor *)(op)) @@ -271,12 +262,12 @@ _bz2_BZ2Compressor_compress_impl(BZ2Compressor *self, Py_buffer *data) { PyObject *result = NULL; - ACQUIRE_LOCK(self); + PyMutex_Lock(&self->mutex); if (self->flushed) PyErr_SetString(PyExc_ValueError, "Compressor has been flushed"); else result = compress(self, data->buf, data->len, BZ_RUN); - RELEASE_LOCK(self); + PyMutex_Unlock(&self->mutex); return result; } @@ -296,14 +287,14 @@ _bz2_BZ2Compressor_flush_impl(BZ2Compressor *self) { PyObject *result = NULL; - ACQUIRE_LOCK(self); + PyMutex_Lock(&self->mutex); if (self->flushed) PyErr_SetString(PyExc_ValueError, "Repeated call to flush()"); else { self->flushed = 1; result = compress(self, NULL, 0, BZ_FINISH); } - RELEASE_LOCK(self); + PyMutex_Unlock(&self->mutex); return result; } @@ -357,13 +348,7 @@ _bz2_BZ2Compressor_impl(PyTypeObject *type, int compresslevel) return NULL; } - self->lock = PyThread_allocate_lock(); - if (self->lock == NULL) { - Py_DECREF(self); - PyErr_SetString(PyExc_MemoryError, "Unable to allocate lock"); - return NULL; - } - + self->mutex = (PyMutex){0}; self->bzs.opaque = NULL; self->bzs.bzalloc = BZ2_Malloc; self->bzs.bzfree = BZ2_Free; @@ -382,10 +367,8 @@ static void BZ2Compressor_dealloc(PyObject *op) { BZ2Compressor *self = _BZ2Compressor_CAST(op); + assert(!PyMutex_IsLocked(&self->mutex)); BZ2_bzCompressEnd(&self->bzs); - if (self->lock != NULL) { - PyThread_free_lock(self->lock); - } PyTypeObject *tp = Py_TYPE(self); tp->tp_free((PyObject *)self); Py_DECREF(tp); @@ -619,12 +602,12 @@ _bz2_BZ2Decompressor_decompress_impl(BZ2Decompressor *self, Py_buffer *data, { PyObject *result = NULL; - ACQUIRE_LOCK(self); + PyMutex_Lock(&self->mutex); if (self->eof) PyErr_SetString(PyExc_EOFError, "End of stream already reached"); else result = decompress(self, data->buf, data->len, max_length); - RELEASE_LOCK(self); + PyMutex_Unlock(&self->mutex); return result; } @@ -650,13 +633,7 @@ _bz2_BZ2Decompressor_impl(PyTypeObject *type) return NULL; } - self->lock = PyThread_allocate_lock(); - if (self->lock == NULL) { - Py_DECREF(self); - PyErr_SetString(PyExc_MemoryError, "Unable to allocate lock"); - return NULL; - } - + self->mutex = (PyMutex){0}; self->needs_input = 1; self->bzs_avail_in_real = 0; self->input_buffer = NULL; @@ -678,15 +655,13 @@ static void BZ2Decompressor_dealloc(PyObject *op) { BZ2Decompressor *self = _BZ2Decompressor_CAST(op); + assert(!PyMutex_IsLocked(&self->mutex)); if(self->input_buffer != NULL) { PyMem_Free(self->input_buffer); } BZ2_bzDecompressEnd(&self->bzs); Py_CLEAR(self->unused_data); - if (self->lock != NULL) { - PyThread_free_lock(self->lock); - } PyTypeObject *tp = Py_TYPE(self); tp->tp_free((PyObject *)self); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index f9f14322df0a5e..6ebd9ebdfce1bb 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -11,7 +11,6 @@ #include "pycore_audit.h" // _PySys_Audit() #include "pycore_backoff.h" #include "pycore_cell.h" // PyCell_GetRef() -#include "pycore_ceval.h" #include "pycore_code.h" #include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS #include "pycore_function.h" @@ -5362,10 +5361,6 @@ dummy_func( tier2 op(_MAKE_WARM, (--)) { current_executor->vm_data.warm = true; - // It's okay if this ends up going negative. - if (--tstate->interp->trace_run_counter == 0) { - _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT); - } } tier2 op(_FATAL_ERROR, (--)) { diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index 6bf64868cbb2d3..9b6506ac3326b3 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -1398,7 +1398,7 @@ _Py_HandlePending(PyThreadState *tstate) if ((breaker & _PY_EVAL_JIT_INVALIDATE_COLD_BIT) != 0) { _Py_unset_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT); _Py_Executors_InvalidateCold(tstate->interp); - tstate->interp->trace_run_counter = JIT_CLEANUP_THRESHOLD; + tstate->interp->executor_creation_counter = JIT_CLEANUP_THRESHOLD; } /* GIL drop request */ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 0e4d86463761a0..9ce0a9f8a4d87b 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7409,9 +7409,6 @@ case _MAKE_WARM: { current_executor->vm_data.warm = true; - if (--tstate->interp->trace_run_counter == 0) { - _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT); - } break; } diff --git a/Python/import.c b/Python/import.c index d01c4d478283ff..45206b46793846 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2358,6 +2358,7 @@ create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) for (struct _inittab *p = INITTAB; p->name != NULL; p++) { if (_PyUnicode_EqualToASCIIString(info.name, p->name)) { found = p; + break; } } if (found == NULL) { diff --git a/Python/optimizer.c b/Python/optimizer.c index 6ad9124744859a..f44f8a9614b846 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -6,6 +6,7 @@ #include "pycore_interp.h" #include "pycore_backoff.h" #include "pycore_bitutils.h" // _Py_popcount32() +#include "pycore_ceval.h" // _Py_set_eval_breaker_bit #include "pycore_code.h" // _Py_GetBaseCodeUnit #include "pycore_function.h" // _PyFunction_LookupByVersion() #include "pycore_interpframe.h" @@ -1343,6 +1344,14 @@ uop_optimize( return -1; } assert(length <= UOP_MAX_TRACE_LENGTH); + + // Check executor coldness + PyThreadState *tstate = PyThreadState_Get(); + // It's okay if this ends up going negative. + if (--tstate->interp->executor_creation_counter == 0) { + _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT); + } + *exec_ptr = executor; return 1; } diff --git a/Python/pystate.c b/Python/pystate.c index 5d0927c6c08196..2141e842a37d2f 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -571,7 +571,7 @@ init_interpreter(PyInterpreterState *interp, interp->executor_list_head = NULL; interp->executor_deletion_list_head = NULL; interp->executor_deletion_list_remaining_capacity = 0; - interp->trace_run_counter = JIT_CLEANUP_THRESHOLD; + interp->executor_creation_counter = JIT_CLEANUP_THRESHOLD; if (interp != &runtime->_main_interpreter) { /* Fix the self-referential, statically initialized fields. */ interp->dtoa = (struct _dtoa_state)_dtoa_state_INIT(interp);