From d7f0214f133442f8f5d4d64044a370e94a5906e8 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 26 Nov 2025 13:50:03 +0100 Subject: [PATCH 1/8] gh-140550: PEP 793 reference documentation (GH-141197) * gh-140550: PEP 793 reference documentation Since the PEP calls for soft-deprecation of the existing initialization function, this reorganizes the relevant docs to put the new way of doing things first, and de-emphasize the old. Some bits, like the tutorial, are left out of this patch. (See the issue for a list.) --- Doc/c-api/extension-modules.rst | 243 +++++--- Doc/c-api/import.rst | 2 +- Doc/c-api/init.rst | 3 +- Doc/c-api/module.rst | 699 +++++++++++++++++------- Doc/c-api/structures.rst | 2 + Doc/c-api/type.rst | 33 +- Doc/data/refcounts.dat | 19 + Doc/howto/free-threading-extensions.rst | 19 +- Doc/tools/.nitignore | 1 - 9 files changed, 736 insertions(+), 285 deletions(-) diff --git a/Doc/c-api/extension-modules.rst b/Doc/c-api/extension-modules.rst index 3d331e6ec12f76..0ce173b4bfea7c 100644 --- a/Doc/c-api/extension-modules.rst +++ b/Doc/c-api/extension-modules.rst @@ -8,7 +8,8 @@ Defining extension modules A C extension for CPython is a shared library (for example, a ``.so`` file on Linux, ``.pyd`` DLL on Windows), which is loadable into the Python process (for example, it is compiled with compatible compiler settings), and which -exports an :ref:`initialization function `. +exports an :dfn:`export hook` function (or an +old-style :ref:`initialization function `). To be importable by default (that is, by :py:class:`importlib.machinery.ExtensionFileLoader`), @@ -23,25 +24,127 @@ and must be named after the module name plus an extension listed in One suitable tool is Setuptools, whose documentation can be found at https://setuptools.pypa.io/en/latest/setuptools.html. -Normally, the initialization function returns a module definition initialized -using :c:func:`PyModuleDef_Init`. -This allows splitting the creation process into several phases: +.. _extension-export-hook: + +Extension export hook +..................... + +.. versionadded:: next + + Support for the :samp:`PyModExport_{}` export hook was added in Python + 3.15. The older way of defining modules is still available: consult either + the :ref:`extension-pyinit` section or earlier versions of this + documentation if you plan to support earlier Python versions. + +The export hook must be an exported function with the following signature: + +.. c:function:: PyModuleDef_Slot *PyModExport_modulename(void) + +For modules with ASCII-only names, the :ref:`export hook ` +must be named :samp:`PyModExport_{}`, +with ```` replaced by the module's name. + +For non-ASCII module names, the export hook must instead be named +:samp:`PyModExportU_{}` (note the ``U``), with ```` encoded using +Python's *punycode* encoding with hyphens replaced by underscores. In Python: + +.. code-block:: python + + def hook_name(name): + try: + suffix = b'_' + name.encode('ascii') + except UnicodeEncodeError: + suffix = b'U_' + name.encode('punycode').replace(b'-', b'_') + return b'PyModExport' + suffix + +The export hook returns an array of :c:type:`PyModuleDef_Slot` entries, +terminated by an entry with a slot ID of ``0``. +These slots describe how the module should be created and initialized. + +This array must remain valid and constant until interpreter shutdown. +Typically, it should use ``static`` storage. +Prefer using the :c:macro:`Py_mod_create` and :c:macro:`Py_mod_exec` slots +for any dynamic behavior. + +The export hook may return ``NULL`` with an exception set to signal failure. + +It is recommended to define the export hook function using a helper macro: + +.. c:macro:: PyMODEXPORT_FUNC + + Declare an extension module export hook. + This macro: + + * specifies the :c:expr:`PyModuleDef_Slot*` return type, + * adds any special linkage declarations required by the platform, and + * for C++, declares the function as ``extern "C"``. +For example, a module called ``spam`` would be defined like this:: + + PyABIInfo_VAR(abi_info); + + static PyModuleDef_Slot spam_slots[] = { + {Py_mod_abi, &abi_info}, + {Py_mod_name, "spam"}, + {Py_mod_init, spam_init_function}, + ... + {0, NULL}, + }; + + PyMODEXPORT_FUNC + PyModExport_spam(void) + { + return spam_slots; + } + +The export hook is typically the only non-\ ``static`` +item defined in the module's C source. + +The hook should be kept short -- ideally, one line as above. +If you do need to use Python C API in this function, it is recommended to call +``PyABIInfo_Check(&abi_info, "modulename")`` first to raise an exception, +rather than crash, in common cases of ABI mismatch. + + +.. note:: + + It is possible to export multiple modules from a single shared library by + defining multiple export hooks. + However, importing them requires a custom importer or suitably named + copies/links of the extension file, because Python's import machinery only + finds the function corresponding to the filename. + See the `Multiple modules in one library `__ + section in :pep:`489` for details. + + +.. _multi-phase-initialization: + +Multi-phase initialization +.......................... + +The process of creating an extension module follows several phases: + +- Python finds and calls the export hook to get information on how to + create the module. - Before any substantial code is executed, Python can determine which capabilities the module supports, and it can adjust the environment or refuse loading an incompatible extension. -- By default, Python itself creates the module object -- that is, it does - the equivalent of :py:meth:`object.__new__` for classes. - It also sets initial attributes like :attr:`~module.__package__` and - :attr:`~module.__loader__`. -- Afterwards, the module object is initialized using extension-specific - code -- the equivalent of :py:meth:`~object.__init__` on classes. + Slots like :c:data:`Py_mod_abi`, :c:data:`Py_mod_gil` and + :c:data:`Py_mod_multiple_interpreters` influence this step. +- By default, Python itself then creates the module object -- that is, it does + the equivalent of calling :py:meth:`~object.__new__` when creating an object. + This step can be overridden using the :c:data:`Py_mod_create` slot. +- Python sets initial module attributes like :attr:`~module.__package__` and + :attr:`~module.__loader__`, and inserts the module object into + :py:attr:`sys.modules`. +- Afterwards, the module object is initialized in an extension-specific way + -- the equivalent of :py:meth:`~object.__init__` when creating an object, + or of executing top-level code in a Python-language module. + The behavior is specified using the :c:data:`Py_mod_exec` slot. This is called *multi-phase initialization* to distinguish it from the legacy -(but still supported) *single-phase initialization* scheme, -where the initialization function returns a fully constructed module. -See the :ref:`single-phase-initialization section below ` -for details. +(but still supported) :ref:`single-phase initialization `, +where an initialization function returns a fully constructed module. .. versionchanged:: 3.5 @@ -53,7 +156,7 @@ Multiple module instances By default, extension modules are not singletons. For example, if the :py:attr:`sys.modules` entry is removed and the module -is re-imported, a new module object is created, and typically populated with +is re-imported, a new module object is created and, typically, populated with fresh method and type objects. The old module is subject to normal garbage collection. This mirrors the behavior of pure-Python modules. @@ -83,36 +186,34 @@ A module may also be limited to the main interpreter using the :c:data:`Py_mod_multiple_interpreters` slot. -.. _extension-export-hook: +.. _extension-pyinit: -Initialization function -....................... +``PyInit`` function +................... -The initialization function defined by an extension module has the -following signature: +.. deprecated:: next + + This functionality is :term:`soft deprecated`. + It will not get new features, but there are no plans to remove it. + +Instead of :c:func:`PyModExport_modulename`, an extension module can define +an older-style :dfn:`initialization function` with the signature: .. c:function:: PyObject* PyInit_modulename(void) Its name should be :samp:`PyInit_{}`, with ```` replaced by the name of the module. +For non-ASCII module names, use :samp:`PyInitU_{}` instead, with +```` encoded in the same way as for the +:ref:`export hook ` (that is, using Punycode +with underscores). -For modules with ASCII-only names, the function must instead be named -:samp:`PyInit_{}`, with ```` replaced by the name of the module. -When using :ref:`multi-phase-initialization`, non-ASCII module names -are allowed. In this case, the initialization function name is -:samp:`PyInitU_{}`, with ```` encoded using Python's -*punycode* encoding with hyphens replaced by underscores. In Python: +If a module exports both :samp:`PyInit_{}` and +:samp:`PyModExport_{}`, the :samp:`PyInit_{}` function +is ignored. -.. code-block:: python - - def initfunc_name(name): - try: - suffix = b'_' + name.encode('ascii') - except UnicodeEncodeError: - suffix = b'U_' + name.encode('punycode').replace(b'-', b'_') - return b'PyInit' + suffix - -It is recommended to define the initialization function using a helper macro: +Like with :c:macro:`PyMODEXPORT_FUNC`, it is recommended to define the +initialization function using a helper macro: .. c:macro:: PyMODINIT_FUNC @@ -123,43 +224,15 @@ It is recommended to define the initialization function using a helper macro: * adds any special linkage declarations required by the platform, and * for C++, declares the function as ``extern "C"``. -For example, a module called ``spam`` would be defined like this:: - - static struct PyModuleDef spam_module = { - .m_base = PyModuleDef_HEAD_INIT, - .m_name = "spam", - ... - }; - - PyMODINIT_FUNC - PyInit_spam(void) - { - return PyModuleDef_Init(&spam_module); - } - -It is possible to export multiple modules from a single shared library by -defining multiple initialization functions. However, importing them requires -using symbolic links or a custom importer, because by default only the -function corresponding to the filename is found. -See the `Multiple modules in one library `__ -section in :pep:`489` for details. - -The initialization function is typically the only non-\ ``static`` -item defined in the module's C source. +Normally, the initialization function (``PyInit_modulename``) returns +a :c:type:`PyModuleDef` instance with non-``NULL`` +:c:member:`~PyModuleDef.m_slots`. This allows Python to use +:ref:`multi-phase initialization `. -.. _multi-phase-initialization: - -Multi-phase initialization -.......................... - -Normally, the :ref:`initialization function ` -(``PyInit_modulename``) returns a :c:type:`PyModuleDef` instance with -non-``NULL`` :c:member:`~PyModuleDef.m_slots`. Before it is returned, the ``PyModuleDef`` instance must be initialized using the following function: - .. c:function:: PyObject* PyModuleDef_Init(PyModuleDef *def) Ensure a module definition is a properly initialized Python object that @@ -167,7 +240,8 @@ using the following function: Return *def* cast to ``PyObject*``, or ``NULL`` if an error occurred. - Calling this function is required for :ref:`multi-phase-initialization`. + Calling this function is required before returning a :c:type:`PyModuleDef` + from a module initialization function. It should not be used in other contexts. Note that Python assumes that ``PyModuleDef`` structures are statically @@ -178,18 +252,37 @@ using the following function: .. versionadded:: 3.5 +For example, a module called ``spam`` would be defined like this:: + + static struct PyModuleDef spam_module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "spam", + ... + }; + + PyMODINIT_FUNC + PyInit_spam(void) + { + return PyModuleDef_Init(&spam_module); + } + + .. _single-phase-initialization: Legacy single-phase initialization -.................................. +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. attention:: - Single-phase initialization is a legacy mechanism to initialize extension +.. deprecated:: next + + Single-phase initialization is :term:`soft deprecated`. + It is a legacy mechanism to initialize extension modules, with known drawbacks and design flaws. Extension module authors are encouraged to use multi-phase initialization instead. -In single-phase initialization, the -:ref:`initialization function ` (``PyInit_modulename``) + However, there are no plans to remove support for it. + +In single-phase initialization, the old-style +:ref:`initializaton function ` (``PyInit_modulename``) should create, populate and return a module object. This is typically done using :c:func:`PyModule_Create` and functions like :c:func:`PyModule_AddObjectRef`. @@ -242,6 +335,8 @@ in the following ways: * Single-phase modules support module lookup functions like :c:func:`PyState_FindModule`. +* The module's :c:member:`PyModuleDef.m_slots` must be NULL. + .. [#testsinglephase] ``_testsinglephase`` is an internal module used in CPython's self-test suite; your installation may or may not include it. diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index ca2e575bcc017a..1786ac6b503895 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -353,7 +353,7 @@ Importing Modules the same as for :c:func:`PyImport_AppendInittab`. On success, create and return a module object. - This module will not be initialized; call :c:func:`!PyModule_Exec` + This module will not be initialized; call :c:func:`PyModule_Exec` to initialize it. (Custom importers should do this in their :py:meth:`~importlib.abc.Loader.exec_module` method.) diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 70643bc07f61d0..7411644f9e110b 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -1717,7 +1717,8 @@ function. You can create and destroy them using the following functions: Only C-level static and global variables are shared between these module objects. - * For modules using single-phase initialization, + * For modules using legacy + :ref:`single-phase initialization `, e.g. :c:func:`PyModule_Create`, the first time a particular extension is imported, it is initialized normally, and a (shallow) copy of its module's dictionary is squirreled away. diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 5d91de48d0d8a5..a12f6331c85912 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -3,11 +3,10 @@ .. _moduleobjects: Module Objects --------------- +============== .. index:: pair: object; module - .. c:var:: PyTypeObject PyModule_Type .. index:: single: ModuleType (in module types) @@ -97,13 +96,6 @@ Module Objects Note that Python code may rename a module by setting its :py:attr:`~module.__name__` attribute. -.. c:function:: void* PyModule_GetState(PyObject *module) - - Return the "state" of the module, that is, a pointer to the block of memory - allocated at module creation time, or ``NULL``. See - :c:member:`PyModuleDef.m_size`. - - .. c:function:: PyModuleDef* PyModule_GetDef(PyObject *module) Return a pointer to the :c:type:`PyModuleDef` struct from which the module was @@ -141,180 +133,188 @@ Module Objects unencodable filenames, use :c:func:`PyModule_GetFilenameObject` instead. -.. _pymoduledef: +.. _pymoduledef_slot: -Module definitions ------------------- +Module definition +----------------- -The functions in the previous section work on any module object, including -modules imported from Python code. +Modules created using the C API are typically defined using an +array of :dfn:`slots`. +The slots provide a "description" of how a module should be created. -Modules defined using the C API typically use a *module definition*, -:c:type:`PyModuleDef` -- a statically allocated, constant “description" of -how a module should be created. +.. versionchanged:: next -The definition is usually used to define an extension's “main” module object -(see :ref:`extension-modules` for details). -It is also used to -:ref:`create extension modules dynamically `. + Previously, a :c:type:`PyModuleDef` struct was necessary to define modules. + The older way of defining modules is still available: consult either the + :ref:`pymoduledef` section or earlier versions of this documentation + if you plan to support earlier Python versions. -Unlike :c:func:`PyModule_New`, the definition allows management of -*module state* -- a piece of memory that is allocated and cleared together -with the module object. -Unlike the module's Python attributes, Python code cannot replace or delete -data stored in module state. +The slots array is usually used to define an extension module's “main” +module object (see :ref:`extension-modules` for details). +It can also be used to +:ref:`create extension modules dynamically `. -.. c:type:: PyModuleDef +Unless specified otherwise, the same slot ID may not be repeated +in an array of slots. - The module definition struct, which holds all information needed to create - a module object. - This structure must be statically allocated (or be otherwise guaranteed - to be valid while any modules created from it exist). - Usually, there is only one variable of this type for each extension module. - .. c:member:: PyModuleDef_Base m_base +.. c:type:: PyModuleDef_Slot - Always initialize this member to :c:macro:`PyModuleDef_HEAD_INIT`. + .. c:member:: int slot - .. c:member:: const char *m_name + A slot ID, chosen from the available ``Py_mod_*`` values explained below. - Name for the new module. + An ID of 0 marks the end of a :c:type:`!PyModuleDef_Slot` array. - .. c:member:: const char *m_doc + .. c:member:: void* value - Docstring for the module; usually a docstring variable created with - :c:macro:`PyDoc_STRVAR` is used. + Value of the slot, whose meaning depends on the slot ID. - .. c:member:: Py_ssize_t m_size + The value may not be NULL. + To leave a slot out, omit the :c:type:`PyModuleDef_Slot` entry entirely. - Module state may be kept in a per-module memory area that can be - retrieved with :c:func:`PyModule_GetState`, rather than in static globals. - This makes modules safe for use in multiple sub-interpreters. + .. versionadded:: 3.5 - This memory area is allocated based on *m_size* on module creation, - and freed when the module object is deallocated, after the - :c:member:`~PyModuleDef.m_free` function has been called, if present. - Setting it to a non-negative value means that the module can be - re-initialized and specifies the additional amount of memory it requires - for its state. +Metadata slots +.............. - Setting ``m_size`` to ``-1`` means that the module does not support - sub-interpreters, because it has global state. - Negative ``m_size`` is only allowed when using - :ref:`legacy single-phase initialization ` - or when :ref:`creating modules dynamically `. +.. c:macro:: Py_mod_name - See :PEP:`3121` for more details. + :c:type:`Slot ID ` for the name of the new module, + as a NUL-terminated UTF8-encoded ``const char *``. - .. c:member:: PyMethodDef* m_methods + Note that modules are typically created using a + :py:class:`~importlib.machinery.ModuleSpec`, and when they are, the + name from the spec will be used instead of :c:data:`!Py_mod_name`. + However, it is still recommended to include this slot for introspection + and debugging purposes. - A pointer to a table of module-level functions, described by - :c:type:`PyMethodDef` values. Can be ``NULL`` if no functions are present. + .. versionadded:: next - .. c:member:: PyModuleDef_Slot* m_slots + Use :c:member:`PyModuleDef.m_name` instead to support previous versions. - An array of slot definitions for multi-phase initialization, terminated by - a ``{0, NULL}`` entry. - When using legacy single-phase initialization, *m_slots* must be ``NULL``. +.. c:macro:: Py_mod_doc - .. versionchanged:: 3.5 + :c:type:`Slot ID ` for the docstring of the new + module, as a NUL-terminated UTF8-encoded ``const char *``. - Prior to version 3.5, this member was always set to ``NULL``, - and was defined as: + Usually it is set to a variable created with :c:macro:`PyDoc_STRVAR`. - .. c:member:: inquiry m_reload + .. versionadded:: next - .. c:member:: traverseproc m_traverse + Use :c:member:`PyModuleDef.m_doc` instead to support previous versions. - A traversal function to call during GC traversal of the module object, or - ``NULL`` if not needed. - This function is not called if the module state was requested but is not - allocated yet. This is the case immediately after the module is created - and before the module is executed (:c:data:`Py_mod_exec` function). More - precisely, this function is not called if :c:member:`~PyModuleDef.m_size` is greater - than 0 and the module state (as returned by :c:func:`PyModule_GetState`) - is ``NULL``. +Feature slots +............. - .. versionchanged:: 3.9 - No longer called before the module state is allocated. +.. c:macro:: Py_mod_abi + + :c:type:`Slot ID ` whose value points to + a :c:struct:`PyABIInfo` structure describing the ABI that + the extension is using. + + A suitable :c:struct:`!PyABIInfo` variable can be defined using the + :c:macro:`PyABIInfo_VAR` macro, as in: - .. c:member:: inquiry m_clear + .. code-block:: c - A clear function to call during GC clearing of the module object, or - ``NULL`` if not needed. + PyABIInfo_VAR(abi_info); - This function is not called if the module state was requested but is not - allocated yet. This is the case immediately after the module is created - and before the module is executed (:c:data:`Py_mod_exec` function). More - precisely, this function is not called if :c:member:`~PyModuleDef.m_size` is greater - than 0 and the module state (as returned by :c:func:`PyModule_GetState`) - is ``NULL``. + static PyModuleDef_Slot mymodule_slots[] = { + {Py_mod_abi, &abi_info}, + ... + }; - Like :c:member:`PyTypeObject.tp_clear`, this function is not *always* - called before a module is deallocated. For example, when reference - counting is enough to determine that an object is no longer used, - the cyclic garbage collector is not involved and - :c:member:`~PyModuleDef.m_free` is called directly. + When creating a module, Python checks the value of this slot + using :c:func:`PyABIInfo_Check`. - .. versionchanged:: 3.9 - No longer called before the module state is allocated. + .. versionadded:: 3.15 - .. c:member:: freefunc m_free +.. c:macro:: Py_mod_multiple_interpreters - A function to call during deallocation of the module object, or ``NULL`` - if not needed. + :c:type:`Slot ID ` whose value is one of: - This function is not called if the module state was requested but is not - allocated yet. This is the case immediately after the module is created - and before the module is executed (:c:data:`Py_mod_exec` function). More - precisely, this function is not called if :c:member:`~PyModuleDef.m_size` is greater - than 0 and the module state (as returned by :c:func:`PyModule_GetState`) - is ``NULL``. + .. c:namespace:: NULL - .. versionchanged:: 3.9 - No longer called before the module state is allocated. + .. c:macro:: Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED + The module does not support being imported in subinterpreters. -Module slots -............ + .. c:macro:: Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED -.. c:type:: PyModuleDef_Slot + The module supports being imported in subinterpreters, + but only when they share the main interpreter's GIL. + (See :ref:`isolating-extensions-howto`.) - .. c:member:: int slot + .. c:macro:: Py_MOD_PER_INTERPRETER_GIL_SUPPORTED - A slot ID, chosen from the available values explained below. + The module supports being imported in subinterpreters, + even when they have their own GIL. + (See :ref:`isolating-extensions-howto`.) - .. c:member:: void* value + This slot determines whether or not importing this module + in a subinterpreter will fail. - Value of the slot, whose meaning depends on the slot ID. + If ``Py_mod_multiple_interpreters`` is not specified, the import + machinery defaults to ``Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED``. - .. versionadded:: 3.5 + .. versionadded:: 3.12 -The available slot types are: +.. c:macro:: Py_mod_gil + + :c:type:`Slot ID ` whose value is one of: + + .. c:namespace:: NULL + + .. c:macro:: Py_MOD_GIL_USED + + The module depends on the presence of the global interpreter lock (GIL), + and may access global state without synchronization. + + .. c:macro:: Py_MOD_GIL_NOT_USED + + The module is safe to run without an active GIL. + + This slot is ignored by Python builds not configured with + :option:`--disable-gil`. Otherwise, it determines whether or not importing + this module will cause the GIL to be automatically enabled. See + :ref:`whatsnew313-free-threaded-cpython` for more detail. + + If ``Py_mod_gil`` is not specified, the import machinery defaults to + ``Py_MOD_GIL_USED``. + + .. versionadded:: 3.13 + + +Creation and initialization slots +................................. .. c:macro:: Py_mod_create - Specifies a function that is called to create the module object itself. - The *value* pointer of this slot must point to a function of the signature: + :c:type:`Slot ID ` for a function that creates + the module object itself. + The function must have the signature: .. c:function:: PyObject* create_module(PyObject *spec, PyModuleDef *def) :no-index-entry: :no-contents-entry: - The function receives a :py:class:`~importlib.machinery.ModuleSpec` - instance, as defined in :PEP:`451`, and the module definition. - It should return a new module object, or set an error + The function will be called with: + + - *spec*: a ``ModuleSpec``-like object, meaning that any attributes defined + for :py:class:`importlib.machinery.ModuleSpec` have matching semantics. + However, any of the attributes may be missing. + - *def*: ``NULL``, or the module definition if the module is created from one. + + The function should return a new module object, or set an error and return ``NULL``. This function should be kept minimal. In particular, it should not call arbitrary Python code, as trying to import the same module again may result in an infinite loop. - Multiple ``Py_mod_create`` slots may not be specified in one module - definition. - If ``Py_mod_create`` is not specified, the import machinery will create a normal module object using :c:func:`PyModule_New`. The name is taken from *spec*, not the definition, to allow extension modules to dynamically adjust @@ -322,122 +322,442 @@ The available slot types are: names through symlinks, all while sharing a single module definition. There is no requirement for the returned object to be an instance of - :c:type:`PyModule_Type`. Any type can be used, as long as it supports - setting and getting import-related attributes. - However, only ``PyModule_Type`` instances may be returned if the - ``PyModuleDef`` has non-``NULL`` ``m_traverse``, ``m_clear``, - ``m_free``; non-zero ``m_size``; or slots other than ``Py_mod_create``. + :c:type:`PyModule_Type`. + However, some slots may only be used with + :c:type:`!PyModule_Type` instances; in particular: + + - :c:macro:`Py_mod_exec`, + - :ref:`module state slots ` (``Py_mod_state_*``), + - :c:macro:`Py_mod_token`. .. versionadded:: 3.5 + .. versionchanged:: next + + The *slots* argument may be a ``ModuleSpec``-like object, rather than + a true :py:class:`~importlib.machinery.ModuleSpec` instance. + Note that previous versions of CPython did not enforce this. + + The *def* argument may now be ``NULL``, since modules are not necessarily + made from definitions. + .. c:macro:: Py_mod_exec - Specifies a function that is called to *execute* the module. - This is equivalent to executing the code of a Python module: typically, - this function adds classes and constants to the module. + :c:type:`Slot ID ` for a function that will + :dfn:`execute`, or initialize, the module. + This function does the equivalent to executing the code of a Python module: + typically, it adds classes and constants to the module. The signature of the function is: .. c:function:: int exec_module(PyObject* module) :no-index-entry: :no-contents-entry: - If multiple ``Py_mod_exec`` slots are specified, they are processed in the - order they appear in the *m_slots* array. + See the :ref:`capi-module-support-functions` section for some useful + functions to call. + + For backwards compatibility, the :c:type:`PyModuleDef.m_slots` array may + contain multiple :c:macro:`!Py_mod_exec` slots; these are processed in the + order they appear in the array. + Elsewhere (that is, in arguments to :c:func:`PyModule_FromSlotsAndSpec` + and in return values of :samp:`PyModExport_{}`), repeating the slot + is not allowed. .. versionadded:: 3.5 -.. c:macro:: Py_mod_multiple_interpreters + .. versionchanged:: next - Specifies one of the following values: + Repeated ``Py_mod_exec`` slots are disallowed, except in + :c:type:`PyModuleDef.m_slots`. - .. c:namespace:: NULL +.. c:macro:: Py_mod_methods - .. c:macro:: Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED + :c:type:`Slot ID ` for a table of module-level + functions, as an array of :c:type:`PyMethodDef` values suitable as the + *functions* argument to :c:func:`PyModule_AddFunctions`. - The module does not support being imported in subinterpreters. + Like other slot IDs, a slots array may only contain one + :c:macro:`!Py_mod_methods` entry. + To add functions from multiple :c:type:`PyMethodDef` arrays, call + :c:func:`PyModule_AddFunctions` in the :c:macro:`Py_mod_exec` function. - .. c:macro:: Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED + The table must be statically allocated (or otherwise guaranteed to outlive + the module object). - The module supports being imported in subinterpreters, - but only when they share the main interpreter's GIL. - (See :ref:`isolating-extensions-howto`.) + .. versionadded:: next - .. c:macro:: Py_MOD_PER_INTERPRETER_GIL_SUPPORTED + Use :c:member:`PyModuleDef.m_methods` instead to support previous versions. - The module supports being imported in subinterpreters, - even when they have their own GIL. - (See :ref:`isolating-extensions-howto`.) +.. _ext-module-state: - This slot determines whether or not importing this module - in a subinterpreter will fail. +Module state +------------ - Multiple ``Py_mod_multiple_interpreters`` slots may not be specified - in one module definition. +Extension modules can have *module state* -- a +piece of memory that is allocated on module creation, +and freed when the module object is deallocated. +The module state is specified using :ref:`dedicated slots `. - If ``Py_mod_multiple_interpreters`` is not specified, the import - machinery defaults to ``Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED``. +A typical use of module state is storing an exception type -- or indeed *any* +type object defined by the module -- - .. versionadded:: 3.12 +Unlike the module's Python attributes, Python code cannot replace or delete +data stored in module state. -.. c:macro:: Py_mod_gil +Keeping per-module information in attributes and module state, rather than in +static globals, makes module objects *isolated* and safer for use in +multiple sub-interpreters. +It also helps Python do an orderly clean-up when it shuts down. - Specifies one of the following values: +Extensions that keep references to Python objects as part of module state must +implement :c:macro:`Py_mod_state_traverse` and :c:macro:`Py_mod_state_clear` +functions to avoid reference leaks. - .. c:namespace:: NULL +To retrieve the state from a given module, use the following functions: - .. c:macro:: Py_MOD_GIL_USED +.. c:function:: void* PyModule_GetState(PyObject *module) - The module depends on the presence of the global interpreter lock (GIL), - and may access global state without synchronization. + Return the "state" of the module, that is, a pointer to the block of memory + allocated at module creation time, or ``NULL``. See + :c:macro:`Py_mod_state_size`. - .. c:macro:: Py_MOD_GIL_NOT_USED + On error, return ``NULL`` with an exception set. + Use :c:func:`PyErr_Occurred` to tell this case apart from missing + module state. - The module is safe to run without an active GIL. - This slot is ignored by Python builds not configured with - :option:`--disable-gil`. Otherwise, it determines whether or not importing - this module will cause the GIL to be automatically enabled. See - :ref:`whatsnew313-free-threaded-cpython` for more detail. +.. c:function:: int PyModule_GetStateSize(PyObject *, Py_ssize_t *result) - Multiple ``Py_mod_gil`` slots may not be specified in one module definition. + Set *\*result* to the size of the module's state, as specified using + :c:macro:`Py_mod_state_size` (or :c:member:`PyModuleDef.m_size`), + and return 0. - If ``Py_mod_gil`` is not specified, the import machinery defaults to - ``Py_MOD_GIL_USED``. + On error, set *\*result* to -1, and return -1 with an exception set. - .. versionadded:: 3.13 + .. versionadded:: next -.. c:macro:: Py_mod_abi - A pointer to a :c:struct:`PyABIInfo` structure that describes the ABI that - the extension is using. - When the module is loaded, the :c:struct:`!PyABIInfo` in this slot is checked - using :c:func:`PyABIInfo_Check`. +.. _ext-module-state-slots: - A suitable :c:struct:`!PyABIInfo` variable can be defined using the - :c:macro:`PyABIInfo_VAR` macro, as in: +Slots for defining module state +............................... - .. code-block:: c +The following :c:member:`PyModuleDef_Slot.slot` IDs are available for +defining the module state. - PyABIInfo_VAR(abi_info); +.. c:macro:: Py_mod_state_size - static PyModuleDef_Slot mymodule_slots[] = { - {Py_mod_abi, &abi_info}, - ... - }; + :c:type:`Slot ID ` for the size of the module state, + in bytes. - .. versionadded:: 3.15 + Setting the value to a non-negative value means that the module can be + re-initialized and specifies the additional amount of memory it requires + for its state. + See :PEP:`3121` for more details. -.. _moduledef-dynamic: + Use :c:func:`PyModule_GetStateSize` to retrieve the size of a given module. + + .. versionadded:: next + + Use :c:member:`PyModuleDef.m_size` instead to support previous versions. + +.. c:macro:: Py_mod_state_traverse + + :c:type:`Slot ID ` for a traversal function to call + during GC traversal of the module object. + + The signature of the function, and meanings of the arguments, + is similar as for :c:member:`PyTypeObject.tp_traverse`: + + .. c:function:: int traverse_module_state(PyObject *module, visitproc visit, void *arg) + :no-index-entry: + :no-contents-entry: + + This function is not called if the module state was requested but is not + allocated yet. This is the case immediately after the module is created + and before the module is executed (:c:data:`Py_mod_exec` function). More + precisely, this function is not called if the state size + (:c:data:`Py_mod_state_size`) is greater than 0 and the module state + (as returned by :c:func:`PyModule_GetState`) is ``NULL``. + + .. versionadded:: next + + Use :c:member:`PyModuleDef.m_size` instead to support previous versions. + +.. c:macro:: Py_mod_state_clear + + :c:type:`Slot ID ` for a clear function to call + during GC clearing of the module object. + + The signature of the function is: + + .. c:function:: int clear_module_state(PyObject* module) + :no-index-entry: + :no-contents-entry: + + This function is not called if the module state was requested but is not + allocated yet. This is the case immediately after the module is created + and before the module is executed (:c:data:`Py_mod_exec` function). More + precisely, this function is not called if the state size + (:c:data:`Py_mod_state_size`) is greater than 0 and the module state + (as returned by :c:func:`PyModule_GetState`) is ``NULL``. + + Like :c:member:`PyTypeObject.tp_clear`, this function is not *always* + called before a module is deallocated. For example, when reference + counting is enough to determine that an object is no longer used, + the cyclic garbage collector is not involved and + the :c:macro:`Py_mod_state_free` function is called directly. + + .. versionadded:: next + + Use :c:member:`PyModuleDef.m_clear` instead to support previous versions. + +.. c:macro:: Py_mod_state_free + + :c:type:`Slot ID ` for a function to call during + deallocation of the module object. + + The signature of the function is: + + .. c:function:: int free_module_state(PyObject* module) + :no-index-entry: + :no-contents-entry: + + This function is not called if the module state was requested but is not + allocated yet. This is the case immediately after the module is created + and before the module is executed (:c:data:`Py_mod_exec` function). More + precisely, this function is not called if the state size + (:c:data:`Py_mod_state_size`) is greater than 0 and the module state + (as returned by :c:func:`PyModule_GetState`) is ``NULL``. + + .. versionadded:: next + + Use :c:member:`PyModuleDef.m_free` instead to support previous versions. + + +.. _ext-module-token: + +Module token +............ + +Each module may have an associated *token*: a pointer-sized value intended to +identify of the module state's memory layout. +This means that if you have a module object, but you are not sure if it +“belongs” to your extension, you can check using code like this: + +.. code-block:: c + + PyObject *module = + + void *module_token; + if (PyModule_GetToken(module, &module_token) < 0) { + return NULL; + } + if (module_token != your_token) { + PyErr_SetString(PyExc_ValueError, "unexpected module") + return NULL; + } + + // This module's state has the expected memory layout; it's safe to cast + struct my_state state = (struct my_state*)PyModule_GetState(module) + +A module's token -- and the *your_token* value to use in the above code -- is: + +- For modules created with :c:type:`PyModuleDef`: the address of that + :c:type:`PyModuleDef`; +- For modules defined with the :c:macro:`Py_mod_token` slot: the value + of that slot; +- For modules created from an ``PyModExport_*`` + :ref:`export hook `: the slots array that the export + hook returned (unless overriden with :c:macro:`Py_mod_token`). + +.. c:macro:: Py_mod_token + + :c:type:`Slot ID ` for the module token. + + If you use this slot to set the module token (rather than rely on the + default), you must ensure that: + + * The pointer outlives the class, so it's not reused for something else + while the class exists. + * It "belongs" to the extension module where the class lives, so it will not + clash with other extensions. + * If the token points to a :c:type:`PyModuleDef` struct, the module should + behave as if it was created from that :c:type:`PyModuleDef`. + In particular, the module state must have matching layout and semantics. + + Modules created from :c:type:`PyModuleDef` allways use the address of + the :c:type:`PyModuleDef` as the token. + This means that :c:macro:`!Py_mod_token` cannot be used in + :c:member:`PyModuleDef.m_slots`. + + .. versionadded:: next + +.. c:function:: int PyModule_GetToken(PyObject *module, void** result) + + Set *\*result* to the module's token and return 0. + + On error, set *\*result* to NULL, and return -1 with an exception set. + + .. versionadded:: next + +See also :c:func:`PyType_GetModuleByToken`. + + +.. _module-from-slots: Creating extension modules dynamically -------------------------------------- -The following functions may be used to create a module outside of an -extension's :ref:`initialization function `. -They are also used in -:ref:`single-phase initialization `. +The following functions may be used to create an extension module dynamically, +rather than from an extension's :ref:`export hook `. + +.. c:function:: PyObject *PyModule_FromSlotsAndSpec(const PyModuleDef_Slot *slots, PyObject *spec) + + Create a new module object, given an array of :ref:`slots ` + and the :py:class:`~importlib.machinery.ModuleSpec` *spec*. + + The *slots* argument must point to an array of :c:type:`PyModuleDef_Slot` + structures, terminated by an entry slot with slot ID of 0 + (typically written as ``{0}`` or ``{0, NULL}`` in C). + The *slots* argument may not be ``NULL``. + + The *spec* argument may be any ``ModuleSpec``-like object, as described + in :c:macro:`Py_mod_create` documentation. + Currently, the *spec* must have a ``name`` attribute. + + On success, return the new module. + On error, return ``NULL`` with an exception set. + + Note that this does not process the module's execution slot + (:c:data:`Py_mod_exec`). + Both :c:func:`!PyModule_FromSlotsAndSpec` and :c:func:`PyModule_Exec` + must be called to fully initialize a module. + (See also :ref:`multi-phase-initialization`.) + + The *slots* array only needs to be valid for the duration of the + :c:func:`!PyModule_FromSlotsAndSpec` call. + In particular, it may be heap-allocated. + + .. versionadded:: next + +.. c:function:: int PyModule_Exec(PyObject *module) + + Execute the :c:data:`Py_mod_exec` slot(s) of the given *module*. + + On success, return 0. + On error, return -1 with an exception set. + + For clarity: If *module* has no slots, for example if it uses + :ref:`legacy single-phase initialization `, + this function does nothing and returns 0. + + .. versionadded:: next + + + +.. _pymoduledef: + +Module definition struct +------------------------ + +Traditionally, extension modules were defined using a *module definition* +as the “description" of how a module should be created. +Rather than using an array of :ref:`slots ` directly, +the definition has dedicated members for most common functionality, +and allows additional slots as an extension mechanism. + +This way of defining modules is still available and there are no plans to +remove it. + +.. c:type:: PyModuleDef + + The module definition struct, which holds information needed to create + a module object. + + This structure must be statically allocated (or be otherwise guaranteed + to be valid while any modules created from it exist). + Usually, there is only one variable of this type for each extension module + defined this way. + + .. c:member:: PyModuleDef_Base m_base + + Always initialize this member to :c:macro:`PyModuleDef_HEAD_INIT`: + + .. c:namespace:: NULL + + .. c:type:: PyModuleDef_Base + + The type of :c:member:`!PyModuleDef.m_base`. + + .. c:macro:: PyModuleDef_HEAD_INIT + + The required initial value for :c:member:`!PyModuleDef.m_base`. + + .. c:member:: const char *m_name + + Corresponds to the :c:macro:`Py_mod_name` slot. + + .. c:member:: const char *m_doc + + These members correspond to the :c:macro:`Py_mod_doc` slot. + Setting this to NULL is equivalent to omitting the slot. + + .. c:member:: Py_ssize_t m_size + + Corresponds to the :c:macro:`Py_mod_state_size` slot. + Setting this to zero is equivalent to omitting the slot. + + When using :ref:`legacy single-phase initialization ` + or when creating modules dynamically using :c:func:`PyModule_Create` + or :c:func:`PyModule_Create2`, :c:member:`!m_size` may be set to -1. + This indicates that the module does not support sub-interpreters, + because it has global state. + + .. c:member:: PyMethodDef *m_methods + + Corresponds to the :c:macro:`Py_mod_methods` slot. + Setting this to NULL is equivalent to omitting the slot. + + .. c:member:: PyModuleDef_Slot* m_slots + + An array of additional slots, terminated by a ``{0, NULL}`` entry. + + This array may not contain slots corresponding to :c:type:`PyModuleDef` + members. + For example, you cannot use :c:macro:`Py_mod_name` in :c:member:`!m_slots`; + the module name must be given as :c:member:`PyModuleDef.m_name`. + + .. versionchanged:: 3.5 + + Prior to version 3.5, this member was always set to ``NULL``, + and was defined as: + + .. c:member:: inquiry m_reload + + .. c:member:: traverseproc m_traverse + inquiry m_clear + freefunc m_free + + These members correspond to the :c:macro:`Py_mod_state_traverse`, + :c:macro:`Py_mod_state_clear`, and :c:macro:`Py_mod_state_free` slots, + respectively. + + Setting these members to NULL is equivalent to omitting the + corresponding slots. + + .. versionchanged:: 3.9 + + :c:member:`m_traverse`, :c:member:`m_clear` and :c:member:`m_free` + functions are longer called before the module state is allocated. + + +.. _moduledef-dynamic: + +The following API can be used to create modules from a :c:type:`!PyModuleDef` +struct: .. c:function:: PyObject* PyModule_Create(PyModuleDef *def) @@ -514,12 +834,13 @@ They are also used in useful for versioning. This may change in the future. +.. _capi-module-support-functions: + Support functions ----------------- -The following functions are provided to help initialize a module -state. -They are intended for a module's execution slots (:c:data:`Py_mod_exec`), +The following functions are provided to help initialize a module object. +They are intended for a module's execution slot (:c:data:`Py_mod_exec`), the initialization function for legacy :ref:`single-phase initialization `, or code that creates modules dynamically. diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index b4e7cb1d77e1a3..62f45def04f746 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -280,6 +280,8 @@ Implementing functions and methods Name of the method. + A ``NULL`` *ml_name* marks the end of a :c:type:`!PyMethodDef` array. + .. c:member:: PyCFunction ml_meth Pointer to the C implementation. diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 2f2060d0582251..1f57cc04f5dc27 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -283,8 +283,8 @@ Type Objects ``Py_TYPE(self)`` may be a *subclass* of the intended class, and subclasses are not necessarily defined in the same module as their superclass. See :c:type:`PyCMethod` to get the class that defines the method. - See :c:func:`PyType_GetModuleByDef` for cases when :c:type:`!PyCMethod` cannot - be used. + See :c:func:`PyType_GetModuleByToken` for cases when :c:type:`!PyCMethod` + cannot be used. .. versionadded:: 3.9 @@ -304,10 +304,10 @@ Type Objects .. versionadded:: 3.9 -.. c:function:: PyObject* PyType_GetModuleByDef(PyTypeObject *type, struct PyModuleDef *def) +.. c:function:: PyObject* PyType_GetModuleByToken(PyTypeObject *type, const void *mod_token) - Find the first superclass whose module was created from - the given :c:type:`PyModuleDef` *def*, and return that module. + Find the first superclass whose module has the given + :ref:`module token `, and return that module. If no module is found, raises a :py:class:`TypeError` and returns ``NULL``. @@ -317,6 +317,23 @@ Type Objects and other places where a method's defining class cannot be passed using the :c:type:`PyCMethod` calling convention. + .. versionadded:: next + + +.. c:function:: PyObject* PyType_GetModuleByDef(PyTypeObject *type, struct PyModuleDef *def) + + Find the first superclass whose module was created from the given + :c:type:`PyModuleDef` *def*, or whose :ref:`module token ` + is equal to *def*, and return that module. + + Note that modules created from a :c:type:`PyModuleDef` always have their + token set to the :c:type:`PyModuleDef`'s address. + In other words, this function is equivalent to + :c:func:`PyType_GetModuleByToken`, except that it: + + - returns a borrowed reference, and + - has a non-``void*`` argument type (which is a cosmetic difference in C). + The returned reference is :term:`borrowed ` from *type*, and will be valid as long as you hold a reference to *type*. Do not release it with :c:func:`Py_DECREF` or similar. @@ -324,10 +341,10 @@ Type Objects .. versionadded:: 3.11 -.. c:function:: int PyType_GetBaseByToken(PyTypeObject *type, void *token, PyTypeObject **result) +.. c:function:: int PyType_GetBaseByToken(PyTypeObject *type, void *tp_token, PyTypeObject **result) Find the first superclass in *type*'s :term:`method resolution order` whose - :c:macro:`Py_tp_token` token is equal to the given one. + :c:macro:`Py_tp_token` token is equal to *tp_token*. * If found, set *\*result* to a new :term:`strong reference` to it and return ``1``. @@ -338,7 +355,7 @@ Type Objects The *result* argument may be ``NULL``, in which case *\*result* is not set. Use this if you need only the return value. - The *token* argument may not be ``NULL``. + The *tp_token* argument may not be ``NULL``. .. versionadded:: 3.14 diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index 48f4f4919e8966..64399f6ab1ff26 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -1472,6 +1472,9 @@ PyModule_Create2:PyObject*::+1: PyModule_Create2:PyModuleDef*:def:: PyModule_Create2:int:module_api_version:: +PyModule_Exec:int::: +PyModule_ExecDef:PyObject*:module:0: + PyModule_ExecDef:int::: PyModule_ExecDef:PyObject*:module:0: PyModule_ExecDef:PyModuleDef*:def:: @@ -1485,6 +1488,10 @@ PyModule_FromDefAndSpec2:PyModuleDef*:def:: PyModule_FromDefAndSpec2:PyObject*:spec:0: PyModule_FromDefAndSpec2:int:module_api_version:: +PyModule_FromSlotsAndSpec:PyObject*::+1: +PyModule_FromSlotsAndSpec:const PyModuleDef_Slot *:slots:: +PyModule_FromSlotsAndSpec:PyObject*:spec:0: + PyModule_GetDef:PyModuleDef*::0: PyModule_GetDef:PyObject*:module:0: @@ -1506,6 +1513,14 @@ PyModule_GetNameObject:PyObject*:module:0: PyModule_GetState:void*::: PyModule_GetState:PyObject*:module:0: +PyModule_GetStateSize:int::: +PyModule_GetStateSize:PyObject*:module:0: +PyModule_GetToken:Py_ssize_t**:result:: + +PyModule_GetToken:int::: +PyModule_GetToken:PyObject*:module:0: +PyModule_GetToken:void**:result:: + PyModule_New:PyObject*::+1: PyModule_New:char*:name:: @@ -2412,6 +2427,10 @@ PyType_GetFlags:PyTypeObject*:type:0: PyType_GetName:PyObject*::+1: PyType_GetName:PyTypeObject*:type:0: +PyType_GetModuleByToken:PyObject*::+1: +PyType_GetModuleByToken:PyTypeObject*:type:0: +PyType_GetModuleByToken:PyModuleDef*:def:: + PyType_GetModuleByDef:PyObject*::0: PyType_GetModuleByDef:PyTypeObject*:type:0: PyType_GetModuleByDef:PyModuleDef*:def:: diff --git a/Doc/howto/free-threading-extensions.rst b/Doc/howto/free-threading-extensions.rst index 5647ab2d87c79c..83eba8cfea3969 100644 --- a/Doc/howto/free-threading-extensions.rst +++ b/Doc/howto/free-threading-extensions.rst @@ -45,9 +45,12 @@ single-phase initialization. Multi-Phase Initialization .......................... -Extensions that use multi-phase initialization (i.e., -:c:func:`PyModuleDef_Init`) should add a :c:data:`Py_mod_gil` slot in the -module definition. If your extension supports older versions of CPython, +Extensions that use :ref:`multi-phase initialization ` +(functions like :c:func:`PyModuleDef_Init`, +:c:func:`PyModExport_* ` export hook, +:c:func:`PyModule_FromSlotsAndSpec`) should add a +:c:data:`Py_mod_gil` slot in the module definition. +If your extension supports older versions of CPython, you should guard the slot with a :c:data:`PY_VERSION_HEX` check. :: @@ -60,18 +63,12 @@ you should guard the slot with a :c:data:`PY_VERSION_HEX` check. {0, NULL} }; - static struct PyModuleDef moduledef = { - PyModuleDef_HEAD_INIT, - .m_slots = module_slots, - ... - }; - Single-Phase Initialization ........................... -Extensions that use single-phase initialization (i.e., -:c:func:`PyModule_Create`) should call :c:func:`PyUnstable_Module_SetGIL` to +Extensions that use legacy :ref:`single-phase initialization ` +(that is, :c:func:`PyModule_Create`) should call :c:func:`PyUnstable_Module_SetGIL` to indicate that they support running with the GIL disabled. The function is only defined in the free-threaded build, so you should guard the call with ``#ifdef Py_GIL_DISABLED`` to avoid compilation errors in the regular build. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 520e64f5fe13b9..c41c70f0ed3306 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -6,7 +6,6 @@ Doc/c-api/descriptor.rst Doc/c-api/float.rst Doc/c-api/init_config.rst Doc/c-api/intro.rst -Doc/c-api/module.rst Doc/c-api/stable.rst Doc/c-api/typeobj.rst Doc/library/ast.rst From 27f62eb711720c215f9798ae30728ee5a1b4d442 Mon Sep 17 00:00:00 2001 From: Itamar Oren Date: Wed, 26 Nov 2025 05:12:49 -0800 Subject: [PATCH 2/8] gh-140011: Delete importdl assertion that prevents importing embedded modules from packages (GH-141605) --- Lib/test/test_embed.py | 54 +++++++++---- Programs/_testembed.c | 173 +++++++++++++++++++++++++++++++++++++++++ Python/importdl.c | 1 - 3 files changed, 212 insertions(+), 16 deletions(-) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 1078796eae84e2..b536794122787d 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -241,21 +241,7 @@ def test_repeated_init_and_inittab(self): def test_create_module_from_initfunc(self): out, err = self.run_embedded_interpreter("test_create_module_from_initfunc") - if support.Py_GIL_DISABLED: - # the test imports a singlephase init extension, so it emits a warning - # under the free-threaded build - expected_runtime_warning = ( - "RuntimeWarning: The global interpreter lock (GIL)" - " has been enabled to load module 'embedded_ext'" - ) - filtered_err_lines = [ - line - for line in err.strip().splitlines() - if expected_runtime_warning not in line - ] - self.assertEqual(filtered_err_lines, []) - else: - self.assertEqual(err, "") + self.assertEqual(self._nogil_filtered_err(err, "embedded_ext"), "") self.assertEqual(out, "\n" "my_test_extension.executed='yes'\n" @@ -264,6 +250,26 @@ def test_create_module_from_initfunc(self): "embedded_ext.executed='yes'\n" ) + def test_inittab_submodule_multiphase(self): + out, err = self.run_embedded_interpreter("test_inittab_submodule_multiphase") + self.assertEqual(err, "") + self.assertEqual(out, + "\n" + "\n" + "Hello from sub-module\n" + "mp_pkg.mp_submod.mp_submod_exec_slot_ran='yes'\n" + "mp_pkg.mp_pkg_exec_slot_ran='yes'\n" + ) + + def test_inittab_submodule_singlephase(self): + out, err = self.run_embedded_interpreter("test_inittab_submodule_singlephase") + self.assertEqual(self._nogil_filtered_err(err, "sp_pkg"), "") + self.assertEqual(out, + "\n" + "\n" + "Hello from sub-module\n" + ) + def test_forced_io_encoding(self): # Checks forced configuration of embedded interpreter IO streams env = dict(os.environ, PYTHONIOENCODING="utf-8:surrogateescape") @@ -541,6 +547,24 @@ def test_getargs_reset_static_parser(self): out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) self.assertEqual(out, '1\n2\n3\n' * INIT_LOOPS) + @staticmethod + def _nogil_filtered_err(err: str, mod_name: str) -> str: + if not support.Py_GIL_DISABLED: + return err + + # the test imports a singlephase init extension, so it emits a warning + # under the free-threaded build + expected_runtime_warning = ( + "RuntimeWarning: The global interpreter lock (GIL)" + f" has been enabled to load module '{mod_name}'" + ) + filtered_err_lines = [ + line + for line in err.strip().splitlines() + if expected_runtime_warning not in line + ] + return "\n".join(filtered_err_lines) + def config_dev_mode(preconfig, config): preconfig['allocator'] = PYMEM_ALLOCATOR_DEBUG diff --git a/Programs/_testembed.c b/Programs/_testembed.c index d0d7d5f03fb9e3..c6a18249e3ccdd 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -2323,6 +2323,177 @@ test_create_module_from_initfunc(void) return Py_RunMain(); } +/// Multi-phase initialization package & submodule /// + +int +mp_pkg_exec(PyObject *mod) +{ + // make this a namespace package + // empty list = namespace package + if (PyModule_Add(mod, "__path__", PyList_New(0)) < 0) { + return -1; + } + if (PyModule_AddStringConstant(mod, "mp_pkg_exec_slot_ran", "yes") < 0) { + return -1; + } + return 0; +} + +static PyModuleDef_Slot mp_pkg_slots[] = { + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {Py_mod_exec, mp_pkg_exec}, + {0, NULL} +}; + +static struct PyModuleDef mp_pkg_def = { + PyModuleDef_HEAD_INIT, + .m_name = "mp_pkg", + .m_size = 0, + .m_slots = mp_pkg_slots, +}; + +PyMODINIT_FUNC +PyInit_mp_pkg(void) +{ + return PyModuleDef_Init(&mp_pkg_def); +} + +static PyObject * +submod_greet(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return PyUnicode_FromString("Hello from sub-module"); +} + +static PyMethodDef submod_methods[] = { + {"greet", submod_greet, METH_NOARGS, NULL}, + {NULL}, +}; + +int +mp_submod_exec(PyObject *mod) +{ + return PyModule_AddStringConstant(mod, "mp_submod_exec_slot_ran", "yes"); +} + +static PyModuleDef_Slot mp_submod_slots[] = { + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {Py_mod_exec, mp_submod_exec}, + {0, NULL} +}; + +static struct PyModuleDef mp_submod_def = { + PyModuleDef_HEAD_INIT, + .m_name = "mp_pkg.mp_submod", + .m_size = 0, + .m_methods = submod_methods, + .m_slots = mp_submod_slots, +}; + +PyMODINIT_FUNC +PyInit_mp_submod(void) +{ + return PyModuleDef_Init(&mp_submod_def); +} + +static int +test_inittab_submodule_multiphase(void) +{ + wchar_t* argv[] = { + PROGRAM_NAME, + L"-c", + L"import sys;" + L"import mp_pkg.mp_submod;" + L"print(mp_pkg.mp_submod);" + L"print(sys.modules['mp_pkg.mp_submod']);" + L"print(mp_pkg.mp_submod.greet());" + L"print(f'{mp_pkg.mp_submod.mp_submod_exec_slot_ran=}');" + L"print(f'{mp_pkg.mp_pkg_exec_slot_ran=}');" + }; + PyConfig config; + if (PyImport_AppendInittab("mp_pkg", + &PyInit_mp_pkg) != 0) { + fprintf(stderr, "PyImport_AppendInittab() failed\n"); + return 1; + } + if (PyImport_AppendInittab("mp_pkg.mp_submod", + &PyInit_mp_submod) != 0) { + fprintf(stderr, "PyImport_AppendInittab() failed\n"); + return 1; + } + PyConfig_InitPythonConfig(&config); + config.isolated = 1; + config_set_argv(&config, Py_ARRAY_LENGTH(argv), argv); + init_from_config_clear(&config); + return Py_RunMain(); +} + +/// Single-phase initialization package & submodule /// + +static struct PyModuleDef sp_pkg_def = { + PyModuleDef_HEAD_INIT, + .m_name = "sp_pkg", + .m_size = 0, +}; + +PyMODINIT_FUNC +PyInit_sp_pkg(void) +{ + PyObject *mod = PyModule_Create(&sp_pkg_def); + if (mod == NULL) { + return NULL; + } + // make this a namespace package + // empty list = namespace package + if (PyModule_Add(mod, "__path__", PyList_New(0)) < 0) { + Py_DECREF(mod); + return NULL; + } + return mod; +} + +static struct PyModuleDef sp_submod_def = { + PyModuleDef_HEAD_INIT, + .m_name = "sp_pkg.sp_submod", + .m_size = 0, + .m_methods = submod_methods, +}; + +PyMODINIT_FUNC +PyInit_sp_submod(void) +{ + return PyModule_Create(&sp_submod_def); +} + +static int +test_inittab_submodule_singlephase(void) +{ + wchar_t* argv[] = { + PROGRAM_NAME, + L"-c", + L"import sys;" + L"import sp_pkg.sp_submod;" + L"print(sp_pkg.sp_submod);" + L"print(sys.modules['sp_pkg.sp_submod']);" + L"print(sp_pkg.sp_submod.greet());" + }; + PyConfig config; + if (PyImport_AppendInittab("sp_pkg", + &PyInit_sp_pkg) != 0) { + fprintf(stderr, "PyImport_AppendInittab() failed\n"); + return 1; + } + if (PyImport_AppendInittab("sp_pkg.sp_submod", + &PyInit_sp_submod) != 0) { + fprintf(stderr, "PyImport_AppendInittab() failed\n"); + return 1; + } + PyConfig_InitPythonConfig(&config); + config.isolated = 1; + config_set_argv(&config, Py_ARRAY_LENGTH(argv), argv); + init_from_config_clear(&config); + return Py_RunMain(); +} + static void wrap_allocator(PyMemAllocatorEx *allocator); static void unwrap_allocator(PyMemAllocatorEx *allocator); @@ -2507,6 +2678,8 @@ static struct TestCase TestCases[] = { {"test_get_incomplete_frame", test_get_incomplete_frame}, {"test_gilstate_after_finalization", test_gilstate_after_finalization}, {"test_create_module_from_initfunc", test_create_module_from_initfunc}, + {"test_inittab_submodule_multiphase", test_inittab_submodule_multiphase}, + {"test_inittab_submodule_singlephase", test_inittab_submodule_singlephase}, {NULL, NULL} }; diff --git a/Python/importdl.c b/Python/importdl.c index 61a9cdaf3754c9..537e8d869dc93c 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -156,7 +156,6 @@ _Py_ext_module_loader_info_init_for_builtin( PyObject *name) { assert(PyUnicode_Check(name)); - assert(PyUnicode_FindChar(name, '.', 0, PyUnicode_GetLength(name), -1) == -1); assert(PyUnicode_GetLength(name) > 0); PyObject *name_encoded = PyUnicode_AsEncodedString(name, "ascii", NULL); From 2ea67caf31453c1755bb5a5c21c5e9477d085deb Mon Sep 17 00:00:00 2001 From: Sergey Miryanov Date: Wed, 26 Nov 2025 19:32:30 +0500 Subject: [PATCH 3/8] GH-141861: Fix TRACE_RECORD if full (GH-141959) --- Lib/test/test_sys.py | 2 +- Python/bytecodes.c | 4 ++-- Python/generated_cases.c.h | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 2e87e38fe5dfdc..04018e9603ff13 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -2254,7 +2254,7 @@ def frame_3_jit() -> None: # 1 extra iteration for tracing. for i in range(_testinternalcapi.TIER2_THRESHOLD + 2): # Careful, doing this in the reverse order breaks tracing: - expected = {enabled} and i >= _testinternalcapi.TIER2_THRESHOLD + 1 + expected = {enabled} and i >= _testinternalcapi.TIER2_THRESHOLD assert sys._jit.is_active() is expected frame_2_jit(expected) assert sys._jit.is_active() is expected diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 6129ea2e723273..01cd1e8359815a 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3018,7 +3018,7 @@ dummy_func( goto stop_tracing; } PyCodeObject *code = _PyFrame_GetCode(frame); - _PyExecutorObject *executor = code->co_executors->executors[this_instr->op.arg]; + _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; assert(executor->vm_data.index == INSTR_OFFSET() - 1); assert(executor->vm_data.code == code); assert(executor->vm_data.valid); @@ -5650,7 +5650,7 @@ dummy_func( LEAVE_TRACING(); int err = stop_tracing_and_jit(tstate, frame); ERROR_IF(err < 0); - DISPATCH_GOTO_NON_TRACING(); + DISPATCH(); } // Super instructions. Instruction deopted. There's a mismatch in what the stack expects // in the optimizer. So we have to reflect in the trace correctly. diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 47805c270f9a0e..57d5e71144d38c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5476,7 +5476,7 @@ JUMP_TO_LABEL(stop_tracing); } PyCodeObject *code = _PyFrame_GetCode(frame); - _PyExecutorObject *executor = code->co_executors->executors[this_instr->op.arg]; + _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; assert(executor->vm_data.index == INSTR_OFFSET() - 1); assert(executor->vm_data.code == code); assert(executor->vm_data.valid); @@ -11714,7 +11714,7 @@ if (err < 0) { JUMP_TO_LABEL(error); } - DISPATCH_GOTO_NON_TRACING(); + DISPATCH(); } _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; if ((_tstate->jit_tracer_state.prev_state.instr->op.code == CALL_LIST_APPEND && From c359ea4c71ace06e4eeec6249660ebaecc1e8e27 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 26 Nov 2025 15:45:06 +0100 Subject: [PATCH 4/8] gh-141909: Correct version where Py_mod_gil was added (GH-141979) --- Doc/data/stable_abi.dat | 2 +- Misc/stable_abi.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 4d9ac8537e2e81..9c5fdcefaf81d0 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -987,7 +987,7 @@ macro,Py_mod_abi,3.15,, macro,Py_mod_create,3.5,, macro,Py_mod_doc,3.15,, macro,Py_mod_exec,3.5,, -macro,Py_mod_gil,3.12,, +macro,Py_mod_gil,3.13,, macro,Py_mod_methods,3.15,, macro,Py_mod_multiple_interpreters,3.12,, macro,Py_mod_name,3.15,, diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 7d156f4b864f68..31d22e64b846ba 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2533,7 +2533,7 @@ [function.PyEval_GetFrameLocals] added = '3.13' [const.Py_mod_gil] - added = '3.12' + added = '3.13' [function.Py_TYPE] # Before 3.14, this was a macro that accessed the PyObject member From 2ff8608b4da33f667960e5099a1a442197acaea4 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 26 Nov 2025 16:10:44 +0100 Subject: [PATCH 5/8] gh-135676: Simplify docs on lexing names (GH-140464) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This simplifies the Lexical Analysis section on Names (but keeps it technically correct) by putting all the info about non-ASCII characters in a separate (and very technical) section. It uses a mental model where the parser doesn't handle Unicode complexity “immediately”, but: - parses any non-ASCII character (outside strings/comments) as part of a name, since these can't (yet) be e.g. operators - normalizes the name - validates the name, using the xid_start/xid_continue sets Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Co-authored-by: Blaise Pabon Co-authored-by: Micha Albert Co-authored-by: KeithTheEE --- Doc/reference/lexical_analysis.rst | 161 ++++++++++++++++++----------- 1 file changed, 103 insertions(+), 58 deletions(-) diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index ea386706e8b511..129dc10d07f7c9 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -386,73 +386,29 @@ Names (identifiers and keywords) :data:`~token.NAME` tokens represent *identifiers*, *keywords*, and *soft keywords*. -Within the ASCII range (U+0001..U+007F), the valid characters for names -include the uppercase and lowercase letters (``A-Z`` and ``a-z``), -the underscore ``_`` and, except for the first character, the digits -``0`` through ``9``. +Names are composed of the following characters: + +* uppercase and lowercase letters (``A-Z`` and ``a-z``), +* the underscore (``_``), +* digits (``0`` through ``9``), which cannot appear as the first character, and +* non-ASCII characters. Valid names may only contain "letter-like" and + "digit-like" characters; see :ref:`lexical-names-nonascii` for details. Names must contain at least one character, but have no upper length limit. Case is significant. -Besides ``A-Z``, ``a-z``, ``_`` and ``0-9``, names can also use "letter-like" -and "number-like" characters from outside the ASCII range, as detailed below. - -All identifiers are converted into the `normalization form`_ NFKC while -parsing; comparison of identifiers is based on NFKC. - -Formally, the first character of a normalized identifier must belong to the -set ``id_start``, which is the union of: - -* Unicode category ```` - uppercase letters (includes ``A`` to ``Z``) -* Unicode category ```` - lowercase letters (includes ``a`` to ``z``) -* Unicode category ```` - titlecase letters -* Unicode category ```` - modifier letters -* Unicode category ```` - other letters -* Unicode category ```` - letter numbers -* {``"_"``} - the underscore -* ```` - an explicit set of characters in `PropList.txt`_ - to support backwards compatibility - -The remaining characters must belong to the set ``id_continue``, which is the -union of: - -* all characters in ``id_start`` -* Unicode category ```` - decimal numbers (includes ``0`` to ``9``) -* Unicode category ```` - connector punctuations -* Unicode category ```` - nonspacing marks -* Unicode category ```` - spacing combining marks -* ```` - another explicit set of characters in - `PropList.txt`_ to support backwards compatibility - -Unicode categories use the version of the Unicode Character Database as -included in the :mod:`unicodedata` module. - -These sets are based on the Unicode standard annex `UAX-31`_. -See also :pep:`3131` for further details. - -Even more formally, names are described by the following lexical definitions: +Formally, names are described by the following lexical definitions: .. grammar-snippet:: :group: python-grammar - NAME: `xid_start` `xid_continue`* - id_start: | | | | | | "_" | - id_continue: `id_start` | | | | | - xid_start: - xid_continue: - identifier: <`NAME`, except keywords> + NAME: `name_start` `name_continue`* + name_start: "a"..."z" | "A"..."Z" | "_" | + name_continue: name_start | "0"..."9" + identifier: <`NAME`, except keywords> -A non-normative listing of all valid identifier characters as defined by -Unicode is available in the `DerivedCoreProperties.txt`_ file in the Unicode -Character Database. - - -.. _UAX-31: https://www.unicode.org/reports/tr31/ -.. _PropList.txt: https://www.unicode.org/Public/17.0.0/ucd/PropList.txt -.. _DerivedCoreProperties.txt: https://www.unicode.org/Public/17.0.0/ucd/DerivedCoreProperties.txt -.. _normalization form: https://www.unicode.org/reports/tr15/#Norm_Forms +Note that not all names matched by this grammar are valid; see +:ref:`lexical-names-nonascii` for details. .. _keywords: @@ -555,6 +511,95 @@ characters: :ref:`atom-identifiers`. +.. _lexical-names-nonascii: + +Non-ASCII characters in names +----------------------------- + +Names that contain non-ASCII characters need additional normalization +and validation beyond the rules and grammar explained +:ref:`above `. +For example, ``ř_1``, ``蛇``, or ``साँप`` are valid names, but ``r〰2``, +``€``, or ``🐍`` are not. + +This section explains the exact rules. + +All names are converted into the `normalization form`_ NFKC while parsing. +This means that, for example, some typographic variants of characters are +converted to their "basic" form. For example, ``fiⁿₐˡᵢᶻₐᵗᵢᵒₙ`` normalizes to +``finalization``, so Python treats them as the same name:: + + >>> fiⁿₐˡᵢᶻₐᵗᵢᵒₙ = 3 + >>> finalization + 3 + +.. note:: + + Normalization is done at the lexical level only. + Run-time functions that take names as *strings* generally do not normalize + their arguments. + For example, the variable defined above is accessible at run time in the + :func:`globals` dictionary as ``globals()["finalization"]`` but not + ``globals()["fiⁿₐˡᵢᶻₐᵗᵢᵒₙ"]``. + +Similarly to how ASCII-only names must contain only letters, digits and +the underscore, and cannot start with a digit, a valid name must +start with a character in the "letter-like" set ``xid_start``, +and the remaining characters must be in the "letter- and digit-like" set +``xid_continue``. + +These sets based on the *XID_Start* and *XID_Continue* sets as defined by the +Unicode standard annex `UAX-31`_. +Python's ``xid_start`` additionally includes the underscore (``_``). +Note that Python does not necessarily conform to `UAX-31`_. + +A non-normative listing of characters in the *XID_Start* and *XID_Continue* +sets as defined by Unicode is available in the `DerivedCoreProperties.txt`_ +file in the Unicode Character Database. +For reference, the construction rules for the ``xid_*`` sets are given below. + +The set ``id_start`` is defined as the union of: + +* Unicode category ```` - uppercase letters (includes ``A`` to ``Z``) +* Unicode category ```` - lowercase letters (includes ``a`` to ``z``) +* Unicode category ```` - titlecase letters +* Unicode category ```` - modifier letters +* Unicode category ```` - other letters +* Unicode category ```` - letter numbers +* {``"_"``} - the underscore +* ```` - an explicit set of characters in `PropList.txt`_ + to support backwards compatibility + +The set ``xid_start`` then closes this set under NFKC normalization, by +removing all characters whose normalization is not of the form +``id_start id_continue*``. + +The set ``id_continue`` is defined as the union of: + +* ``id_start`` (see above) +* Unicode category ```` - decimal numbers (includes ``0`` to ``9``) +* Unicode category ```` - connector punctuations +* Unicode category ```` - nonspacing marks +* Unicode category ```` - spacing combining marks +* ```` - another explicit set of characters in + `PropList.txt`_ to support backwards compatibility + +Again, ``xid_continue`` closes this set under NFKC normalization. + +Unicode categories use the version of the Unicode Character Database as +included in the :mod:`unicodedata` module. + +.. _UAX-31: https://www.unicode.org/reports/tr31/ +.. _PropList.txt: https://www.unicode.org/Public/17.0.0/ucd/PropList.txt +.. _DerivedCoreProperties.txt: https://www.unicode.org/Public/17.0.0/ucd/DerivedCoreProperties.txt +.. _normalization form: https://www.unicode.org/reports/tr15/#Norm_Forms + +.. seealso:: + + * :pep:`3131` -- Supporting Non-ASCII Identifiers + * :pep:`672` -- Unicode-related Security Considerations for Python + + .. _literals: Literals From 2c1fdf3592cee02f7d2e2f250f2fbd67ddea0a2c Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Wed, 26 Nov 2025 07:44:25 -0800 Subject: [PATCH 6/8] gh-141968: Use `bytearray.take_bytes` in `base64` `_b32encode` and `_b32decode` (#141971) --- Lib/base64.py | 4 ++-- .../Library/2025-11-25-22-54-07.gh-issue-141968.vg3AMJ.rst | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-25-22-54-07.gh-issue-141968.vg3AMJ.rst diff --git a/Lib/base64.py b/Lib/base64.py index f95132a4274051..341bf8eaf1891e 100644 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -193,7 +193,7 @@ def _b32encode(alphabet, s): encoded[-3:] = b'===' elif leftover == 4: encoded[-1:] = b'=' - return bytes(encoded) + return encoded.take_bytes() def _b32decode(alphabet, s, casefold=False, map01=None): # Delay the initialization of the table to not waste memory @@ -238,7 +238,7 @@ def _b32decode(alphabet, s, casefold=False, map01=None): last = acc.to_bytes(5) # big endian leftover = (43 - 5 * padchars) // 8 # 1: 4, 3: 3, 4: 2, 6: 1 decoded[-5:] = last[:leftover] - return bytes(decoded) + return decoded.take_bytes() def b32encode(s): diff --git a/Misc/NEWS.d/next/Library/2025-11-25-22-54-07.gh-issue-141968.vg3AMJ.rst b/Misc/NEWS.d/next/Library/2025-11-25-22-54-07.gh-issue-141968.vg3AMJ.rst new file mode 100644 index 00000000000000..8492cb35d5bc76 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-25-22-54-07.gh-issue-141968.vg3AMJ.rst @@ -0,0 +1,2 @@ +Remove a data copy from :func:`base64.b32decode` and +:func:`base64.b32encode` by using :func:`bytearray.take_bytes`. From 9dbf77beb62c65e18eb766f0e867b7d96fa7b56a Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Wed, 26 Nov 2025 07:45:12 -0800 Subject: [PATCH 7/8] gh-141968: use `bytearray.take_bytes` in `wave._byteswap` (#141973) --- Lib/wave.py | 2 +- .../next/Library/2025-11-25-23-22-46.gh-issue-141968.R1sHnJ.rst | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-25-23-22-46.gh-issue-141968.R1sHnJ.rst diff --git a/Lib/wave.py b/Lib/wave.py index 056bd6aab7ffa3..25ca9ef168e8a5 100644 --- a/Lib/wave.py +++ b/Lib/wave.py @@ -97,7 +97,7 @@ def _byteswap(data, width): for j in range(width): swapped_data[i + width - 1 - j] = data[i + j] - return bytes(swapped_data) + return swapped_data.take_bytes() class _Chunk: diff --git a/Misc/NEWS.d/next/Library/2025-11-25-23-22-46.gh-issue-141968.R1sHnJ.rst b/Misc/NEWS.d/next/Library/2025-11-25-23-22-46.gh-issue-141968.R1sHnJ.rst new file mode 100644 index 00000000000000..16b9ad78c3cd3f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-25-23-22-46.gh-issue-141968.R1sHnJ.rst @@ -0,0 +1,2 @@ +Remove data copy from :func:`wave.Wave_read.readframes` and +:func:`wave.Wave_write.writeframes` by using :func:`bytearray.take_bytes`. From 9ac14288d7147dbbae08a8ffd8581e0f5e6fd706 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Wed, 26 Nov 2025 07:46:25 -0800 Subject: [PATCH 8/8] gh-141968: use `bytearray.take_bytes` in `encodings.idna` (#141975) --- Lib/encodings/idna.py | 5 +++-- .../Library/2025-11-25-23-35-07.gh-issue-141968.b3Gscp.rst | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-11-25-23-35-07.gh-issue-141968.b3Gscp.rst diff --git a/Lib/encodings/idna.py b/Lib/encodings/idna.py index 0c90b4c9fe18fa..d31ee07ab45b76 100644 --- a/Lib/encodings/idna.py +++ b/Lib/encodings/idna.py @@ -226,7 +226,8 @@ def encode(self, input, errors='strict'): offset + exc.end, exc.reason, ) - return bytes(result+trailing_dot), len(input) + result += trailing_dot + return result.take_bytes(), len(input) def decode(self, input, errors='strict'): @@ -311,7 +312,7 @@ def _buffer_encode(self, input, errors, final): result += trailing_dot size += len(trailing_dot) - return (bytes(result), size) + return (result.take_bytes(), size) class IncrementalDecoder(codecs.BufferedIncrementalDecoder): def _buffer_decode(self, input, errors, final): diff --git a/Misc/NEWS.d/next/Library/2025-11-25-23-35-07.gh-issue-141968.b3Gscp.rst b/Misc/NEWS.d/next/Library/2025-11-25-23-35-07.gh-issue-141968.b3Gscp.rst new file mode 100644 index 00000000000000..32658be5b2f632 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-25-23-35-07.gh-issue-141968.b3Gscp.rst @@ -0,0 +1,2 @@ +Remove data copy from :mod:`encodings.idna` :meth:`~codecs.Codec.encode` and +:meth:`~codecs.IncrementalEncoder.encode` by using :func:`bytearray.take_bytes`.