diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 382a0abedfc3bc..88813c6f1a4bbf 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -1693,11 +1693,14 @@ inherited by child processes. value is actually a synchronized wrapper for the array. *typecode_or_type* determines the type of the elements of the returned array: - it is either a ctypes type or a one character typecode of the kind used by - the :mod:`array` module. If *size_or_initializer* is an integer, then it - determines the length of the array, and the array will be initially zeroed. - Otherwise, *size_or_initializer* is a sequence which is used to initialize - the array and whose length determines the length of the array. + it is either a :ref:`ctypes type ` or a one + character typecode of the kind used by the :mod:`array` module with the + exception of ``'w'``, which is not supported. In addition, the ``'c'`` + typecode is an alias for :class:`ctypes.c_char`. If *size_or_initializer* + is an integer, then it determines the length of the array, and the array + will be initially zeroed. Otherwise, *size_or_initializer* is a sequence + which is used to initialize the array and whose length determines the length + of the array. If *lock* is ``True`` (the default) then a new lock object is created to synchronize access to the value. If *lock* is a :class:`Lock` or diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index ad8bab0080847d..11a9d76640a439 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1486,9 +1486,9 @@ _PyOpcode_macro_expansion[256] = { [STORE_ATTR_SLOT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_SLOT, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 } } }, [STORE_ATTR_WITH_HINT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_WITH_HINT, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 } } }, [STORE_DEREF] = { .nuops = 1, .uops = { { _STORE_DEREF, OPARG_SIMPLE, 0 } } }, - [STORE_FAST] = { .nuops = 1, .uops = { { _STORE_FAST, OPARG_SIMPLE, 0 } } }, - [STORE_FAST_LOAD_FAST] = { .nuops = 2, .uops = { { _STORE_FAST, OPARG_TOP, 0 }, { _LOAD_FAST, OPARG_BOTTOM, 0 } } }, - [STORE_FAST_STORE_FAST] = { .nuops = 2, .uops = { { _STORE_FAST, OPARG_TOP, 0 }, { _STORE_FAST, OPARG_BOTTOM, 0 } } }, + [STORE_FAST] = { .nuops = 2, .uops = { { _SWAP_FAST, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, + [STORE_FAST_LOAD_FAST] = { .nuops = 3, .uops = { { _SWAP_FAST, OPARG_TOP, 0 }, { _POP_TOP, OPARG_TOP, 0 }, { _LOAD_FAST, OPARG_BOTTOM, 0 } } }, + [STORE_FAST_STORE_FAST] = { .nuops = 4, .uops = { { _SWAP_FAST, OPARG_TOP, 0 }, { _POP_TOP, OPARG_TOP, 0 }, { _SWAP_FAST, OPARG_BOTTOM, 0 }, { _POP_TOP, OPARG_BOTTOM, 0 } } }, [STORE_GLOBAL] = { .nuops = 1, .uops = { { _STORE_GLOBAL, OPARG_SIMPLE, 0 } } }, [STORE_NAME] = { .nuops = 1, .uops = { { _STORE_NAME, OPARG_SIMPLE, 0 } } }, [STORE_SLICE] = { .nuops = 1, .uops = { { _STORE_SLICE, OPARG_SIMPLE, 0 } } }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index fa5355c8dbb5c6..b06f4221c5963b 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -325,24 +325,24 @@ extern "C" { #define _STORE_ATTR_SLOT 533 #define _STORE_ATTR_WITH_HINT 534 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 535 -#define _STORE_FAST_0 536 -#define _STORE_FAST_1 537 -#define _STORE_FAST_2 538 -#define _STORE_FAST_3 539 -#define _STORE_FAST_4 540 -#define _STORE_FAST_5 541 -#define _STORE_FAST_6 542 -#define _STORE_FAST_7 543 #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 544 -#define _STORE_SUBSCR 545 -#define _STORE_SUBSCR_DICT 546 -#define _STORE_SUBSCR_LIST_INT 547 -#define _SWAP 548 -#define _SWAP_2 549 -#define _SWAP_3 550 +#define _STORE_SLICE 535 +#define _STORE_SUBSCR 536 +#define _STORE_SUBSCR_DICT 537 +#define _STORE_SUBSCR_LIST_INT 538 +#define _SWAP 539 +#define _SWAP_2 540 +#define _SWAP_3 541 +#define _SWAP_FAST 542 +#define _SWAP_FAST_0 543 +#define _SWAP_FAST_1 544 +#define _SWAP_FAST_2 545 +#define _SWAP_FAST_3 546 +#define _SWAP_FAST_4 547 +#define _SWAP_FAST_5 548 +#define _SWAP_FAST_6 549 +#define _SWAP_FAST_7 550 #define _TIER2_RESUME_CHECK 551 #define _TO_BOOL 552 #define _TO_BOOL_BOOL TO_BOOL_BOOL @@ -1069,63 +1069,90 @@ extern "C" { #define _STORE_ATTR_SLOT_r21 1263 #define _STORE_ATTR_WITH_HINT_r21 1264 #define _STORE_DEREF_r10 1265 -#define _STORE_FAST_r10 1266 -#define _STORE_FAST_0_r10 1267 -#define _STORE_FAST_1_r10 1268 -#define _STORE_FAST_2_r10 1269 -#define _STORE_FAST_3_r10 1270 -#define _STORE_FAST_4_r10 1271 -#define _STORE_FAST_5_r10 1272 -#define _STORE_FAST_6_r10 1273 -#define _STORE_FAST_7_r10 1274 -#define _STORE_FAST_LOAD_FAST_r11 1275 -#define _STORE_FAST_STORE_FAST_r20 1276 -#define _STORE_GLOBAL_r10 1277 -#define _STORE_NAME_r10 1278 -#define _STORE_SLICE_r30 1279 -#define _STORE_SUBSCR_r30 1280 -#define _STORE_SUBSCR_DICT_r31 1281 -#define _STORE_SUBSCR_LIST_INT_r32 1282 -#define _SWAP_r11 1283 -#define _SWAP_2_r02 1284 -#define _SWAP_2_r12 1285 -#define _SWAP_2_r22 1286 -#define _SWAP_2_r33 1287 -#define _SWAP_3_r03 1288 -#define _SWAP_3_r13 1289 -#define _SWAP_3_r23 1290 -#define _SWAP_3_r33 1291 -#define _TIER2_RESUME_CHECK_r00 1292 -#define _TIER2_RESUME_CHECK_r11 1293 -#define _TIER2_RESUME_CHECK_r22 1294 -#define _TIER2_RESUME_CHECK_r33 1295 -#define _TO_BOOL_r11 1296 -#define _TO_BOOL_BOOL_r01 1297 -#define _TO_BOOL_BOOL_r11 1298 -#define _TO_BOOL_BOOL_r22 1299 -#define _TO_BOOL_BOOL_r33 1300 -#define _TO_BOOL_INT_r11 1301 -#define _TO_BOOL_LIST_r11 1302 -#define _TO_BOOL_NONE_r01 1303 -#define _TO_BOOL_NONE_r11 1304 -#define _TO_BOOL_NONE_r22 1305 -#define _TO_BOOL_NONE_r33 1306 -#define _TO_BOOL_STR_r11 1307 -#define _TRACE_RECORD_r00 1308 -#define _UNARY_INVERT_r11 1309 -#define _UNARY_NEGATIVE_r11 1310 -#define _UNARY_NOT_r01 1311 -#define _UNARY_NOT_r11 1312 -#define _UNARY_NOT_r22 1313 -#define _UNARY_NOT_r33 1314 -#define _UNPACK_EX_r10 1315 -#define _UNPACK_SEQUENCE_r10 1316 -#define _UNPACK_SEQUENCE_LIST_r10 1317 -#define _UNPACK_SEQUENCE_TUPLE_r10 1318 -#define _UNPACK_SEQUENCE_TWO_TUPLE_r12 1319 -#define _WITH_EXCEPT_START_r33 1320 -#define _YIELD_VALUE_r11 1321 -#define MAX_UOP_REGS_ID 1321 +#define _STORE_FAST_LOAD_FAST_r11 1266 +#define _STORE_FAST_STORE_FAST_r20 1267 +#define _STORE_GLOBAL_r10 1268 +#define _STORE_NAME_r10 1269 +#define _STORE_SLICE_r30 1270 +#define _STORE_SUBSCR_r30 1271 +#define _STORE_SUBSCR_DICT_r31 1272 +#define _STORE_SUBSCR_LIST_INT_r32 1273 +#define _SWAP_r11 1274 +#define _SWAP_2_r02 1275 +#define _SWAP_2_r12 1276 +#define _SWAP_2_r22 1277 +#define _SWAP_2_r33 1278 +#define _SWAP_3_r03 1279 +#define _SWAP_3_r13 1280 +#define _SWAP_3_r23 1281 +#define _SWAP_3_r33 1282 +#define _SWAP_FAST_r01 1283 +#define _SWAP_FAST_r11 1284 +#define _SWAP_FAST_r22 1285 +#define _SWAP_FAST_r33 1286 +#define _SWAP_FAST_0_r01 1287 +#define _SWAP_FAST_0_r11 1288 +#define _SWAP_FAST_0_r22 1289 +#define _SWAP_FAST_0_r33 1290 +#define _SWAP_FAST_1_r01 1291 +#define _SWAP_FAST_1_r11 1292 +#define _SWAP_FAST_1_r22 1293 +#define _SWAP_FAST_1_r33 1294 +#define _SWAP_FAST_2_r01 1295 +#define _SWAP_FAST_2_r11 1296 +#define _SWAP_FAST_2_r22 1297 +#define _SWAP_FAST_2_r33 1298 +#define _SWAP_FAST_3_r01 1299 +#define _SWAP_FAST_3_r11 1300 +#define _SWAP_FAST_3_r22 1301 +#define _SWAP_FAST_3_r33 1302 +#define _SWAP_FAST_4_r01 1303 +#define _SWAP_FAST_4_r11 1304 +#define _SWAP_FAST_4_r22 1305 +#define _SWAP_FAST_4_r33 1306 +#define _SWAP_FAST_5_r01 1307 +#define _SWAP_FAST_5_r11 1308 +#define _SWAP_FAST_5_r22 1309 +#define _SWAP_FAST_5_r33 1310 +#define _SWAP_FAST_6_r01 1311 +#define _SWAP_FAST_6_r11 1312 +#define _SWAP_FAST_6_r22 1313 +#define _SWAP_FAST_6_r33 1314 +#define _SWAP_FAST_7_r01 1315 +#define _SWAP_FAST_7_r11 1316 +#define _SWAP_FAST_7_r22 1317 +#define _SWAP_FAST_7_r33 1318 +#define _TIER2_RESUME_CHECK_r00 1319 +#define _TIER2_RESUME_CHECK_r11 1320 +#define _TIER2_RESUME_CHECK_r22 1321 +#define _TIER2_RESUME_CHECK_r33 1322 +#define _TO_BOOL_r11 1323 +#define _TO_BOOL_BOOL_r01 1324 +#define _TO_BOOL_BOOL_r11 1325 +#define _TO_BOOL_BOOL_r22 1326 +#define _TO_BOOL_BOOL_r33 1327 +#define _TO_BOOL_INT_r11 1328 +#define _TO_BOOL_LIST_r11 1329 +#define _TO_BOOL_NONE_r01 1330 +#define _TO_BOOL_NONE_r11 1331 +#define _TO_BOOL_NONE_r22 1332 +#define _TO_BOOL_NONE_r33 1333 +#define _TO_BOOL_STR_r11 1334 +#define _TRACE_RECORD_r00 1335 +#define _UNARY_INVERT_r11 1336 +#define _UNARY_NEGATIVE_r11 1337 +#define _UNARY_NOT_r01 1338 +#define _UNARY_NOT_r11 1339 +#define _UNARY_NOT_r22 1340 +#define _UNARY_NOT_r33 1341 +#define _UNPACK_EX_r10 1342 +#define _UNPACK_SEQUENCE_r10 1343 +#define _UNPACK_SEQUENCE_LIST_r10 1344 +#define _UNPACK_SEQUENCE_TUPLE_r10 1345 +#define _UNPACK_SEQUENCE_TWO_TUPLE_r12 1346 +#define _WITH_EXCEPT_START_r33 1347 +#define _YIELD_VALUE_r11 1348 +#define MAX_UOP_REGS_ID 1348 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 558184f890bf93..0d49110c7e3e0c 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -65,15 +65,15 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_SMALL_INT_2] = 0, [_LOAD_SMALL_INT_3] = 0, [_LOAD_SMALL_INT] = HAS_ARG_FLAG, - [_STORE_FAST_0] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST_1] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST_2] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST_3] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST_4] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST_5] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST_6] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST_7] = HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, - [_STORE_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ESCAPES_FLAG, + [_SWAP_FAST_0] = HAS_LOCAL_FLAG, + [_SWAP_FAST_1] = HAS_LOCAL_FLAG, + [_SWAP_FAST_2] = HAS_LOCAL_FLAG, + [_SWAP_FAST_3] = HAS_LOCAL_FLAG, + [_SWAP_FAST_4] = HAS_LOCAL_FLAG, + [_SWAP_FAST_5] = HAS_LOCAL_FLAG, + [_SWAP_FAST_6] = HAS_LOCAL_FLAG, + [_SWAP_FAST_7] = HAS_LOCAL_FLAG, + [_SWAP_FAST] = HAS_ARG_FLAG | HAS_LOCAL_FLAG, [_POP_TOP] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_POP_TOP_NOP] = 0, [_POP_TOP_INT] = 0, @@ -362,7 +362,7 @@ const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1] = { [_LOAD_FAST] = { 0, 8 }, [_LOAD_FAST_BORROW] = { 0, 8 }, [_LOAD_SMALL_INT] = { 0, 4 }, - [_STORE_FAST] = { 0, 8 }, + [_SWAP_FAST] = { 0, 8 }, [_INIT_CALL_PY_EXACT_ARGS] = { 0, 5 }, [_COPY] = { 1, 4 }, [_SWAP] = { 2, 4 }, @@ -639,85 +639,85 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, - [_STORE_FAST_0] = { - .best = { 1, 1, 1, 1 }, + [_SWAP_FAST_0] = { + .best = { 0, 1, 2, 3 }, .entries = { - { -1, -1, -1 }, - { 0, 1, _STORE_FAST_0_r10 }, - { -1, -1, -1 }, - { -1, -1, -1 }, + { 1, 0, _SWAP_FAST_0_r01 }, + { 1, 1, _SWAP_FAST_0_r11 }, + { 2, 2, _SWAP_FAST_0_r22 }, + { 3, 3, _SWAP_FAST_0_r33 }, }, }, - [_STORE_FAST_1] = { - .best = { 1, 1, 1, 1 }, + [_SWAP_FAST_1] = { + .best = { 0, 1, 2, 3 }, .entries = { - { -1, -1, -1 }, - { 0, 1, _STORE_FAST_1_r10 }, - { -1, -1, -1 }, - { -1, -1, -1 }, + { 1, 0, _SWAP_FAST_1_r01 }, + { 1, 1, _SWAP_FAST_1_r11 }, + { 2, 2, _SWAP_FAST_1_r22 }, + { 3, 3, _SWAP_FAST_1_r33 }, }, }, - [_STORE_FAST_2] = { - .best = { 1, 1, 1, 1 }, + [_SWAP_FAST_2] = { + .best = { 0, 1, 2, 3 }, .entries = { - { -1, -1, -1 }, - { 0, 1, _STORE_FAST_2_r10 }, - { -1, -1, -1 }, - { -1, -1, -1 }, + { 1, 0, _SWAP_FAST_2_r01 }, + { 1, 1, _SWAP_FAST_2_r11 }, + { 2, 2, _SWAP_FAST_2_r22 }, + { 3, 3, _SWAP_FAST_2_r33 }, }, }, - [_STORE_FAST_3] = { - .best = { 1, 1, 1, 1 }, + [_SWAP_FAST_3] = { + .best = { 0, 1, 2, 3 }, .entries = { - { -1, -1, -1 }, - { 0, 1, _STORE_FAST_3_r10 }, - { -1, -1, -1 }, - { -1, -1, -1 }, + { 1, 0, _SWAP_FAST_3_r01 }, + { 1, 1, _SWAP_FAST_3_r11 }, + { 2, 2, _SWAP_FAST_3_r22 }, + { 3, 3, _SWAP_FAST_3_r33 }, }, }, - [_STORE_FAST_4] = { - .best = { 1, 1, 1, 1 }, + [_SWAP_FAST_4] = { + .best = { 0, 1, 2, 3 }, .entries = { - { -1, -1, -1 }, - { 0, 1, _STORE_FAST_4_r10 }, - { -1, -1, -1 }, - { -1, -1, -1 }, + { 1, 0, _SWAP_FAST_4_r01 }, + { 1, 1, _SWAP_FAST_4_r11 }, + { 2, 2, _SWAP_FAST_4_r22 }, + { 3, 3, _SWAP_FAST_4_r33 }, }, }, - [_STORE_FAST_5] = { - .best = { 1, 1, 1, 1 }, + [_SWAP_FAST_5] = { + .best = { 0, 1, 2, 3 }, .entries = { - { -1, -1, -1 }, - { 0, 1, _STORE_FAST_5_r10 }, - { -1, -1, -1 }, - { -1, -1, -1 }, + { 1, 0, _SWAP_FAST_5_r01 }, + { 1, 1, _SWAP_FAST_5_r11 }, + { 2, 2, _SWAP_FAST_5_r22 }, + { 3, 3, _SWAP_FAST_5_r33 }, }, }, - [_STORE_FAST_6] = { - .best = { 1, 1, 1, 1 }, + [_SWAP_FAST_6] = { + .best = { 0, 1, 2, 3 }, .entries = { - { -1, -1, -1 }, - { 0, 1, _STORE_FAST_6_r10 }, - { -1, -1, -1 }, - { -1, -1, -1 }, + { 1, 0, _SWAP_FAST_6_r01 }, + { 1, 1, _SWAP_FAST_6_r11 }, + { 2, 2, _SWAP_FAST_6_r22 }, + { 3, 3, _SWAP_FAST_6_r33 }, }, }, - [_STORE_FAST_7] = { - .best = { 1, 1, 1, 1 }, + [_SWAP_FAST_7] = { + .best = { 0, 1, 2, 3 }, .entries = { - { -1, -1, -1 }, - { 0, 1, _STORE_FAST_7_r10 }, - { -1, -1, -1 }, - { -1, -1, -1 }, + { 1, 0, _SWAP_FAST_7_r01 }, + { 1, 1, _SWAP_FAST_7_r11 }, + { 2, 2, _SWAP_FAST_7_r22 }, + { 3, 3, _SWAP_FAST_7_r33 }, }, }, - [_STORE_FAST] = { - .best = { 1, 1, 1, 1 }, + [_SWAP_FAST] = { + .best = { 0, 1, 2, 3 }, .entries = { - { -1, -1, -1 }, - { 0, 1, _STORE_FAST_r10 }, - { -1, -1, -1 }, - { -1, -1, -1 }, + { 1, 0, _SWAP_FAST_r01 }, + { 1, 1, _SWAP_FAST_r11 }, + { 2, 2, _SWAP_FAST_r22 }, + { 3, 3, _SWAP_FAST_r33 }, }, }, [_POP_TOP] = { @@ -3340,15 +3340,42 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_LOAD_SMALL_INT_r01] = _LOAD_SMALL_INT, [_LOAD_SMALL_INT_r12] = _LOAD_SMALL_INT, [_LOAD_SMALL_INT_r23] = _LOAD_SMALL_INT, - [_STORE_FAST_0_r10] = _STORE_FAST_0, - [_STORE_FAST_1_r10] = _STORE_FAST_1, - [_STORE_FAST_2_r10] = _STORE_FAST_2, - [_STORE_FAST_3_r10] = _STORE_FAST_3, - [_STORE_FAST_4_r10] = _STORE_FAST_4, - [_STORE_FAST_5_r10] = _STORE_FAST_5, - [_STORE_FAST_6_r10] = _STORE_FAST_6, - [_STORE_FAST_7_r10] = _STORE_FAST_7, - [_STORE_FAST_r10] = _STORE_FAST, + [_SWAP_FAST_0_r01] = _SWAP_FAST_0, + [_SWAP_FAST_0_r11] = _SWAP_FAST_0, + [_SWAP_FAST_0_r22] = _SWAP_FAST_0, + [_SWAP_FAST_0_r33] = _SWAP_FAST_0, + [_SWAP_FAST_1_r01] = _SWAP_FAST_1, + [_SWAP_FAST_1_r11] = _SWAP_FAST_1, + [_SWAP_FAST_1_r22] = _SWAP_FAST_1, + [_SWAP_FAST_1_r33] = _SWAP_FAST_1, + [_SWAP_FAST_2_r01] = _SWAP_FAST_2, + [_SWAP_FAST_2_r11] = _SWAP_FAST_2, + [_SWAP_FAST_2_r22] = _SWAP_FAST_2, + [_SWAP_FAST_2_r33] = _SWAP_FAST_2, + [_SWAP_FAST_3_r01] = _SWAP_FAST_3, + [_SWAP_FAST_3_r11] = _SWAP_FAST_3, + [_SWAP_FAST_3_r22] = _SWAP_FAST_3, + [_SWAP_FAST_3_r33] = _SWAP_FAST_3, + [_SWAP_FAST_4_r01] = _SWAP_FAST_4, + [_SWAP_FAST_4_r11] = _SWAP_FAST_4, + [_SWAP_FAST_4_r22] = _SWAP_FAST_4, + [_SWAP_FAST_4_r33] = _SWAP_FAST_4, + [_SWAP_FAST_5_r01] = _SWAP_FAST_5, + [_SWAP_FAST_5_r11] = _SWAP_FAST_5, + [_SWAP_FAST_5_r22] = _SWAP_FAST_5, + [_SWAP_FAST_5_r33] = _SWAP_FAST_5, + [_SWAP_FAST_6_r01] = _SWAP_FAST_6, + [_SWAP_FAST_6_r11] = _SWAP_FAST_6, + [_SWAP_FAST_6_r22] = _SWAP_FAST_6, + [_SWAP_FAST_6_r33] = _SWAP_FAST_6, + [_SWAP_FAST_7_r01] = _SWAP_FAST_7, + [_SWAP_FAST_7_r11] = _SWAP_FAST_7, + [_SWAP_FAST_7_r22] = _SWAP_FAST_7, + [_SWAP_FAST_7_r33] = _SWAP_FAST_7, + [_SWAP_FAST_r01] = _SWAP_FAST, + [_SWAP_FAST_r11] = _SWAP_FAST, + [_SWAP_FAST_r22] = _SWAP_FAST, + [_SWAP_FAST_r33] = _SWAP_FAST, [_POP_TOP_r10] = _POP_TOP, [_POP_TOP_NOP_r00] = _POP_TOP_NOP, [_POP_TOP_NOP_r10] = _POP_TOP_NOP, @@ -4883,24 +4910,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_STORE_ATTR_WITH_HINT_r21] = "_STORE_ATTR_WITH_HINT_r21", [_STORE_DEREF] = "_STORE_DEREF", [_STORE_DEREF_r10] = "_STORE_DEREF_r10", - [_STORE_FAST] = "_STORE_FAST", - [_STORE_FAST_r10] = "_STORE_FAST_r10", - [_STORE_FAST_0] = "_STORE_FAST_0", - [_STORE_FAST_0_r10] = "_STORE_FAST_0_r10", - [_STORE_FAST_1] = "_STORE_FAST_1", - [_STORE_FAST_1_r10] = "_STORE_FAST_1_r10", - [_STORE_FAST_2] = "_STORE_FAST_2", - [_STORE_FAST_2_r10] = "_STORE_FAST_2_r10", - [_STORE_FAST_3] = "_STORE_FAST_3", - [_STORE_FAST_3_r10] = "_STORE_FAST_3_r10", - [_STORE_FAST_4] = "_STORE_FAST_4", - [_STORE_FAST_4_r10] = "_STORE_FAST_4_r10", - [_STORE_FAST_5] = "_STORE_FAST_5", - [_STORE_FAST_5_r10] = "_STORE_FAST_5_r10", - [_STORE_FAST_6] = "_STORE_FAST_6", - [_STORE_FAST_6_r10] = "_STORE_FAST_6_r10", - [_STORE_FAST_7] = "_STORE_FAST_7", - [_STORE_FAST_7_r10] = "_STORE_FAST_7_r10", [_STORE_GLOBAL] = "_STORE_GLOBAL", [_STORE_GLOBAL_r10] = "_STORE_GLOBAL_r10", [_STORE_NAME] = "_STORE_NAME", @@ -4925,6 +4934,51 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_SWAP_3_r13] = "_SWAP_3_r13", [_SWAP_3_r23] = "_SWAP_3_r23", [_SWAP_3_r33] = "_SWAP_3_r33", + [_SWAP_FAST] = "_SWAP_FAST", + [_SWAP_FAST_r01] = "_SWAP_FAST_r01", + [_SWAP_FAST_r11] = "_SWAP_FAST_r11", + [_SWAP_FAST_r22] = "_SWAP_FAST_r22", + [_SWAP_FAST_r33] = "_SWAP_FAST_r33", + [_SWAP_FAST_0] = "_SWAP_FAST_0", + [_SWAP_FAST_0_r01] = "_SWAP_FAST_0_r01", + [_SWAP_FAST_0_r11] = "_SWAP_FAST_0_r11", + [_SWAP_FAST_0_r22] = "_SWAP_FAST_0_r22", + [_SWAP_FAST_0_r33] = "_SWAP_FAST_0_r33", + [_SWAP_FAST_1] = "_SWAP_FAST_1", + [_SWAP_FAST_1_r01] = "_SWAP_FAST_1_r01", + [_SWAP_FAST_1_r11] = "_SWAP_FAST_1_r11", + [_SWAP_FAST_1_r22] = "_SWAP_FAST_1_r22", + [_SWAP_FAST_1_r33] = "_SWAP_FAST_1_r33", + [_SWAP_FAST_2] = "_SWAP_FAST_2", + [_SWAP_FAST_2_r01] = "_SWAP_FAST_2_r01", + [_SWAP_FAST_2_r11] = "_SWAP_FAST_2_r11", + [_SWAP_FAST_2_r22] = "_SWAP_FAST_2_r22", + [_SWAP_FAST_2_r33] = "_SWAP_FAST_2_r33", + [_SWAP_FAST_3] = "_SWAP_FAST_3", + [_SWAP_FAST_3_r01] = "_SWAP_FAST_3_r01", + [_SWAP_FAST_3_r11] = "_SWAP_FAST_3_r11", + [_SWAP_FAST_3_r22] = "_SWAP_FAST_3_r22", + [_SWAP_FAST_3_r33] = "_SWAP_FAST_3_r33", + [_SWAP_FAST_4] = "_SWAP_FAST_4", + [_SWAP_FAST_4_r01] = "_SWAP_FAST_4_r01", + [_SWAP_FAST_4_r11] = "_SWAP_FAST_4_r11", + [_SWAP_FAST_4_r22] = "_SWAP_FAST_4_r22", + [_SWAP_FAST_4_r33] = "_SWAP_FAST_4_r33", + [_SWAP_FAST_5] = "_SWAP_FAST_5", + [_SWAP_FAST_5_r01] = "_SWAP_FAST_5_r01", + [_SWAP_FAST_5_r11] = "_SWAP_FAST_5_r11", + [_SWAP_FAST_5_r22] = "_SWAP_FAST_5_r22", + [_SWAP_FAST_5_r33] = "_SWAP_FAST_5_r33", + [_SWAP_FAST_6] = "_SWAP_FAST_6", + [_SWAP_FAST_6_r01] = "_SWAP_FAST_6_r01", + [_SWAP_FAST_6_r11] = "_SWAP_FAST_6_r11", + [_SWAP_FAST_6_r22] = "_SWAP_FAST_6_r22", + [_SWAP_FAST_6_r33] = "_SWAP_FAST_6_r33", + [_SWAP_FAST_7] = "_SWAP_FAST_7", + [_SWAP_FAST_7_r01] = "_SWAP_FAST_7_r01", + [_SWAP_FAST_7_r11] = "_SWAP_FAST_7_r11", + [_SWAP_FAST_7_r22] = "_SWAP_FAST_7_r22", + [_SWAP_FAST_7_r33] = "_SWAP_FAST_7_r33", [_TIER2_RESUME_CHECK] = "_TIER2_RESUME_CHECK", [_TIER2_RESUME_CHECK_r00] = "_TIER2_RESUME_CHECK_r00", [_TIER2_RESUME_CHECK_r11] = "_TIER2_RESUME_CHECK_r11", @@ -5035,23 +5089,23 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _LOAD_SMALL_INT: return 0; - case _STORE_FAST_0: + case _SWAP_FAST_0: return 1; - case _STORE_FAST_1: + case _SWAP_FAST_1: return 1; - case _STORE_FAST_2: + case _SWAP_FAST_2: return 1; - case _STORE_FAST_3: + case _SWAP_FAST_3: return 1; - case _STORE_FAST_4: + case _SWAP_FAST_4: return 1; - case _STORE_FAST_5: + case _SWAP_FAST_5: return 1; - case _STORE_FAST_6: + case _SWAP_FAST_6: return 1; - case _STORE_FAST_7: + case _SWAP_FAST_7: return 1; - case _STORE_FAST: + case _SWAP_FAST: return 1; case _POP_TOP: return 1; diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index bff97fe8320b22..041a9a37172fa9 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -67,6 +67,9 @@ def iter_ops(ex): def get_ops(ex): return list(iter_ops(ex)) +def count_ops(ex, name): + return len([opname for opname in iter_opnames(ex) if opname == name]) + @requires_specialization @unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds") @@ -1918,8 +1921,8 @@ def testfunc(n): self.assertIn("_BINARY_OP_SUBSCR_STR_INT", uops) self.assertIn("_COMPARE_OP_STR", uops) self.assertIn("_POP_TOP_NOP", uops) - self.assertNotIn("_POP_TOP", uops) - self.assertNotIn("_POP_TOP_INT", uops) + self.assertLessEqual(count_ops(ex, "_POP_TOP"), 2) + self.assertLessEqual(count_ops(ex, "_POP_TOP_INT"), 1) def test_call_type_1_guards_removed(self): def testfunc(n): @@ -2209,10 +2212,9 @@ def testfunc(n): self.assertEqual(res, TIER2_THRESHOLD) self.assertIsNotNone(ex) uops = get_opnames(ex) - pop_tops = [opname for opname in iter_opnames(ex) if opname == "_POP_TOP"] self.assertIn("_CALL_BUILTIN_O", uops) self.assertIn("_POP_TOP_NOP", uops) - self.assertLessEqual(len(pop_tops), 1) + self.assertLessEqual(count_ops(ex, "_POP_TOP"), 4) def test_call_method_descriptor_o(self): def testfunc(n): @@ -2227,10 +2229,9 @@ def testfunc(n): self.assertEqual(res, TIER2_THRESHOLD) self.assertIsNotNone(ex) uops = get_opnames(ex) - pop_tops = [opname for opname in iter_opnames(ex) if opname == "_POP_TOP"] self.assertIn("_CALL_METHOD_DESCRIPTOR_O", uops) self.assertIn("_POP_TOP_NOP", uops) - self.assertLessEqual(len(pop_tops), 1) + self.assertLessEqual(count_ops(ex, "_POP_TOP"), 4) def test_get_len_with_const_tuple(self): def testfunc(n): @@ -2539,6 +2540,47 @@ def f(n): self.assertNotIn("_LOAD_ATTR_METHOD_NO_DICT", uops) self.assertNotIn("_LOAD_ATTR_METHOD_LAZY_DICT", uops) + def test_store_fast_refcount_elimination(self): + def foo(x): + # Since x is known to be + # a constant value (1) here, + # The refcount is eliminated in the STORE_FAST. + x = 2 + return x + def testfunc(n): + # The STORE_FAST for the range here needs a POP_TOP + # (for now, until we do loop peeling). + for _ in range(n): + foo(1) + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_POP_TOP_NOP", uops) + self.assertIn("_PUSH_FRAME", uops) + self.assertLessEqual(count_ops(ex, "_POP_TOP"), 1) + + def test_store_fast_refcount_elimination_when_uninitialized(self): + def foo(): + # Since y is known to be + # uninitialized (NULL) here, + # The refcount is eliminated in the STORE_FAST. + y = 2 + return y + def testfunc(n): + # The STORE_FAST for the range here needs a POP_TOP + # (for now, until we do loop peeling). + for _ in range(n): + foo() + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_POP_TOP_NOP", uops) + self.assertIn("_PUSH_FRAME", uops) + self.assertLessEqual(count_ops(ex, "_POP_TOP"), 1) + + def test_float_op_refcount_elimination(self): def testfunc(args): a, b, n = args @@ -2569,7 +2611,7 @@ class C(): uops = get_opnames(ex) self.assertIn("_LOAD_ATTR_INSTANCE_VALUE", uops) - self.assertNotIn("_POP_TOP", uops) + self.assertLessEqual(count_ops(ex, "_POP_TOP"), 2) self.assertIn("_POP_TOP_NOP", uops) def test_load_attr_with_hint(self): @@ -2590,7 +2632,7 @@ class C: uops = get_opnames(ex) self.assertIn("_LOAD_ATTR_WITH_HINT", uops) - self.assertNotIn("_POP_TOP", uops) + self.assertLessEqual(count_ops(ex, "_POP_TOP"), 2) self.assertIn("_POP_TOP_NOP", uops) def test_load_addr_slot(self): @@ -2609,7 +2651,7 @@ class C: uops = get_opnames(ex) self.assertIn("_LOAD_ATTR_SLOT", uops) - self.assertNotIn("_POP_TOP", uops) + self.assertLessEqual(count_ops(ex, "_POP_TOP"), 2) self.assertIn("_POP_TOP_NOP", uops) def test_int_add_op_refcount_elimination(self): @@ -2625,7 +2667,7 @@ def testfunc(n): uops = get_opnames(ex) self.assertIn("_BINARY_OP_ADD_INT", uops) self.assertIn("_POP_TOP_NOP", uops) - self.assertNotIn("_POP_TOP", uops) + self.assertLessEqual(count_ops(ex, "_POP_TOP"), 2) def test_int_sub_op_refcount_elimination(self): def testfunc(n): @@ -2640,7 +2682,7 @@ def testfunc(n): uops = get_opnames(ex) self.assertIn("_BINARY_OP_SUBTRACT_INT", uops) self.assertIn("_POP_TOP_NOP", uops) - self.assertNotIn("_POP_TOP", uops) + self.assertLessEqual(count_ops(ex, "_POP_TOP"), 2) def test_int_mul_op_refcount_elimination(self): def testfunc(n): @@ -2655,7 +2697,7 @@ def testfunc(n): uops = get_opnames(ex) self.assertIn("_BINARY_OP_MULTIPLY_INT", uops) self.assertIn("_POP_TOP_NOP", uops) - self.assertNotIn("_POP_TOP", uops) + self.assertLessEqual(count_ops(ex, "_POP_TOP"), 2) def test_int_cmp_op_refcount_elimination(self): def testfunc(n): @@ -2670,7 +2712,7 @@ def testfunc(n): uops = get_opnames(ex) self.assertIn("_COMPARE_OP_INT", uops) self.assertIn("_POP_TOP_NOP", uops) - self.assertNotIn("_POP_TOP", uops) + self.assertLessEqual(count_ops(ex, "_POP_TOP"), 2) def test_float_cmp_op_refcount_elimination(self): def testfunc(n): @@ -2685,7 +2727,7 @@ def testfunc(n): uops = get_opnames(ex) self.assertIn("_COMPARE_OP_FLOAT", uops) self.assertIn("_POP_TOP_NOP", uops) - self.assertNotIn("_POP_TOP", uops) + self.assertLessEqual(count_ops(ex, "_POP_TOP"), 2) def test_str_cmp_op_refcount_elimination(self): def testfunc(n): @@ -2700,7 +2742,7 @@ def testfunc(n): uops = get_opnames(ex) self.assertIn("_COMPARE_OP_STR", uops) self.assertIn("_POP_TOP_NOP", uops) - self.assertNotIn("_POP_TOP", uops) + self.assertLessEqual(count_ops(ex, "_POP_TOP"), 2) def test_unicode_add_op_refcount_elimination(self): def testfunc(n): @@ -2716,7 +2758,7 @@ def testfunc(n): uops = get_opnames(ex) self.assertIn("_BINARY_OP_ADD_UNICODE", uops) self.assertIn("_POP_TOP_NOP", uops) - self.assertNotIn("_POP_TOP", uops) + self.assertLessEqual(count_ops(ex, "_POP_TOP"), 2) def test_remove_guard_for_slice_list(self): def f(n): @@ -2775,7 +2817,7 @@ class C: uops = get_opnames(ex) self.assertIn("_STORE_ATTR_INSTANCE_VALUE", uops) - self.assertNotIn("_POP_TOP", uops) + self.assertLessEqual(count_ops(ex, "_POP_TOP"), 1) self.assertIn("_POP_TOP_NOP", uops) def test_store_attr_with_hint(self): @@ -2795,7 +2837,7 @@ class C: uops = get_opnames(ex) self.assertIn("_STORE_ATTR_WITH_HINT", uops) - self.assertNotIn("_POP_TOP", uops) + self.assertLessEqual(count_ops(ex, "_POP_TOP"), 1) self.assertIn("_POP_TOP_NOP", uops) def test_store_subscr_int(self): @@ -2813,7 +2855,7 @@ def testfunc(n): self.assertIsNotNone(ex) uops = get_opnames(ex) self.assertIn("_STORE_SUBSCR_LIST_INT", uops) - self.assertNotIn("_POP_TOP", uops) + self.assertLessEqual(count_ops(ex, "_POP_TOP"), 1) self.assertNotIn("_POP_TOP_INT", uops) self.assertIn("_POP_TOP_NOP", uops) @@ -2850,7 +2892,7 @@ def testfunc(n): self.assertIsNotNone(ex) uops = get_opnames(ex) self.assertIn("_STORE_SUBSCR_DICT", uops) - self.assertNotIn("_POP_TOP", uops) + self.assertLessEqual(count_ops(ex, "_POP_TOP"), 1) self.assertIn("_POP_TOP_NOP", uops) def test_attr_promotion_failure(self): @@ -3183,7 +3225,8 @@ def testfunc(n): assert "_LOAD_GLOBAL_BUILTIN" not in uops assert "_LOAD_CONST_INLINE_BORROW" in uops assert "_POP_TOP_NOP" in uops - assert "_POP_TOP" not in uops + pop_top_count = len([opname for opname in ex if opname == "_POP_TOP" ]) + assert pop_top_count <= 2 """), PYTHON_JIT="1") self.assertEqual(result[0].rc, 0, result) @@ -3214,8 +3257,8 @@ def testfunc(n): uops = get_opnames(ex) self.assertIn("_BINARY_OP_SUBSCR_LIST_INT", uops) - self.assertNotIn("_POP_TOP", uops) - self.assertNotIn("_POP_TOP_INT", uops) + self.assertLessEqual(count_ops(ex, "_POP_TOP"), 2) + self.assertLessEqual(count_ops(ex, "_POP_TOP_INT"), 1) self.assertIn("_POP_TOP_NOP", uops) def test_binary_subscr_tuple_int(self): @@ -3233,8 +3276,8 @@ def testfunc(n): uops = get_opnames(ex) self.assertIn("_BINARY_OP_SUBSCR_TUPLE_INT", uops) - self.assertNotIn("_POP_TOP", uops) - self.assertNotIn("_POP_TOP_INT", uops) + self.assertLessEqual(count_ops(ex, "_POP_TOP"), 3) + self.assertLessEqual(count_ops(ex, "_POP_TOP_INT"), 1) self.assertIn("_POP_TOP_NOP", uops) def test_is_op(self): @@ -3252,7 +3295,7 @@ def test_is_false(n): self.assertIn("_IS_OP", uops) self.assertIn("_POP_TOP_NOP", uops) - self.assertNotIn("_POP_TOP", uops) + self.assertLessEqual(count_ops(ex, "_POP_TOP"), 2) def test_is_true(n): @@ -3268,7 +3311,7 @@ def test_is_true(n): self.assertIn("_IS_OP", uops) self.assertIn("_POP_TOP_NOP", uops) - self.assertNotIn("_POP_TOP", uops) + self.assertLessEqual(count_ops(ex, "_POP_TOP"), 2) def test_is_not(n): @@ -3285,7 +3328,7 @@ def test_is_not(n): self.assertIn("_IS_OP", uops) self.assertIn("_POP_TOP_NOP", uops) - self.assertNotIn("_POP_TOP", uops) + self.assertLessEqual(count_ops(ex, "_POP_TOP"), 2) def test_is_none(n): @@ -3300,7 +3343,7 @@ def test_is_none(n): self.assertIn("_IS_OP", uops) self.assertIn("_POP_TOP_NOP", uops) - self.assertNotIn("_POP_TOP", uops) + self.assertLessEqual(count_ops(ex, "_POP_TOP"), 2) def test_143026(self): # https://github.com/python/cpython/issues/143026 diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-01-02-17-11-16.gh-issue-143361.YDnvdC.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-01-02-17-11-16.gh-issue-143361.YDnvdC.rst new file mode 100644 index 00000000000000..df4e3b61c672cb --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-01-02-17-11-16.gh-issue-143361.YDnvdC.rst @@ -0,0 +1,2 @@ +Add ``PY_VECTORCALL_ARGUMENTS_OFFSET`` to ``_Py_CallBuiltinClass_StackRefSteal`` to +avoid redundant allocations diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 25339e919377e0..366bca44f83f16 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -310,11 +310,13 @@ dummy_func( value = PyStackRef_FromPyObjectBorrow(obj); } - replicate(8) inst(STORE_FAST, (value --)) { + macro(STORE_FAST) = _SWAP_FAST + POP_TOP; + + replicate(8) op(_SWAP_FAST, (value -- trash)) { _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; DEAD(value); - PyStackRef_XCLOSE(tmp); + trash = tmp; } pseudo(STORE_FAST_MAYBE_NULL, (unused --)) = { diff --git a/Python/ceval.c b/Python/ceval.c index 924afaa97443cb..d90ac5c69b9e81 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1273,7 +1273,7 @@ _Py_CallBuiltinClass_StackRefSteal( goto cleanup; } PyTypeObject *tp = (PyTypeObject *)PyStackRef_AsPyObjectBorrow(callable); - res = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); + res = tp->tp_vectorcall((PyObject *)tp, args_o, total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); assert((res != NULL) ^ (PyErr_Occurred() != NULL)); cleanup: diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 38401f40b7a08b..3fe7a3200269c3 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1539,190 +1539,709 @@ break; } - case _STORE_FAST_0_r10: { + case _SWAP_FAST_0_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + oparg = 0; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache0 = trash; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_0_r11: { CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef value; + _PyStackRef trash; _PyStackRef _stack_item_0 = _tos_cache0; oparg = 0; assert(oparg == CURRENT_OPARG()); value = _stack_item_0; _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); + trash = tmp; + _tos_cache0 = trash; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_0_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 0; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_1; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache1 = trash; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_0_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = 0; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_2; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache2 = trash; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_1_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + oparg = 1; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache0 = trash; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _STORE_FAST_1_r10: { + case _SWAP_FAST_1_r11: { CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef value; + _PyStackRef trash; _PyStackRef _stack_item_0 = _tos_cache0; oparg = 1; assert(oparg == CURRENT_OPARG()); value = _stack_item_0; _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); + trash = tmp; + _tos_cache0 = trash; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_1_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 1; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_1; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache1 = trash; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _STORE_FAST_2_r10: { + case _SWAP_FAST_1_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = 1; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_2; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache2 = trash; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_2_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache0 = trash; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_2_r11: { CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef value; + _PyStackRef trash; _PyStackRef _stack_item_0 = _tos_cache0; oparg = 2; assert(oparg == CURRENT_OPARG()); value = _stack_item_0; _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); + trash = tmp; + _tos_cache0 = trash; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_2_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_1; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache1 = trash; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_2_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = 2; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_2; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache2 = trash; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_3_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache0 = trash; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _STORE_FAST_3_r10: { + case _SWAP_FAST_3_r11: { CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef value; + _PyStackRef trash; _PyStackRef _stack_item_0 = _tos_cache0; oparg = 3; assert(oparg == CURRENT_OPARG()); value = _stack_item_0; _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); + trash = tmp; + _tos_cache0 = trash; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_3_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_1; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache1 = trash; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _STORE_FAST_4_r10: { + case _SWAP_FAST_3_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = 3; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_2; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache2 = trash; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_4_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache0 = trash; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_4_r11: { CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef value; + _PyStackRef trash; _PyStackRef _stack_item_0 = _tos_cache0; oparg = 4; assert(oparg == CURRENT_OPARG()); value = _stack_item_0; _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); + trash = tmp; + _tos_cache0 = trash; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_4_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_1; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache1 = trash; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_4_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = 4; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_2; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache2 = trash; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_5_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + oparg = 5; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache0 = trash; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _STORE_FAST_5_r10: { + case _SWAP_FAST_5_r11: { CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef value; + _PyStackRef trash; _PyStackRef _stack_item_0 = _tos_cache0; oparg = 5; assert(oparg == CURRENT_OPARG()); value = _stack_item_0; _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); + trash = tmp; + _tos_cache0 = trash; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_5_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 5; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_1; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache1 = trash; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_5_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = 5; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_2; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache2 = trash; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_6_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache0 = trash; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _STORE_FAST_6_r10: { + case _SWAP_FAST_6_r11: { CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef value; + _PyStackRef trash; _PyStackRef _stack_item_0 = _tos_cache0; oparg = 6; assert(oparg == CURRENT_OPARG()); value = _stack_item_0; _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); + trash = tmp; + _tos_cache0 = trash; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_6_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_1; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache1 = trash; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _STORE_FAST_7_r10: { + case _SWAP_FAST_6_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = 6; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_2; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache2 = trash; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_7_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + value = stack_pointer[-1]; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache0 = trash; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_7_r11: { CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef value; + _PyStackRef trash; _PyStackRef _stack_item_0 = _tos_cache0; oparg = 7; assert(oparg == CURRENT_OPARG()); value = _stack_item_0; _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); + trash = tmp; + _tos_cache0 = trash; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_7_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_1; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache1 = trash; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _STORE_FAST_r10: { + case _SWAP_FAST_7_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = 7; + assert(oparg == CURRENT_OPARG()); + value = _stack_item_2; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache2 = trash; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + oparg = CURRENT_OPARG(); + value = stack_pointer[-1]; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache0 = trash; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_r11: { CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef value; + _PyStackRef trash; _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); value = _stack_item_0; _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); + trash = tmp; + _tos_cache0 = trash; + SET_CURRENT_CACHED_VALUES(1); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + value = _stack_item_1; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache1 = trash; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + break; + } + + case _SWAP_FAST_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _PyStackRef value; + _PyStackRef trash; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + oparg = CURRENT_OPARG(); + value = _stack_item_2; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + _tos_cache2 = trash; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 4f97424c7eab2c..a1d5a98255c405 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -10734,14 +10734,23 @@ next_instr += 1; INSTRUCTION_STATS(STORE_FAST); _PyStackRef value; - value = stack_pointer[-1]; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyStackRef trash; + // _SWAP_FAST + { + value = stack_pointer[-1]; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + trash = tmp; + } + // _POP_TOP + { + value = trash; + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } DISPATCH(); } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 15dd81d9c6e531..b01ed3eaeeacb2 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -109,8 +109,10 @@ dummy_func(void) { o = owner; } - op(_STORE_FAST, (value --)) { + op(_SWAP_FAST, (value -- trash)) { + JitOptRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; + trash = tmp; } op(_STORE_SUBSCR_LIST_INT, (value, list_st, sub_st -- ls, ss)) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index c423a1885243d2..f23769c37c954d 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -99,13 +99,14 @@ break; } - case _STORE_FAST: { + case _SWAP_FAST: { JitOptRef value; + JitOptRef trash; value = stack_pointer[-1]; + JitOptRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = value; - CHECK_STACK_BOUNDS(-1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + trash = tmp; + stack_pointer[-1] = trash; break; } diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 8a71eff465e5a3..c6b54b9b58b795 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -846,8 +846,12 @@ _Py_uop_frame_new( frame->locals[i] = args[i]; } + // If the args are known, then it's safe to just initialize + // every other non-set local to null symbol. + bool default_null = args != NULL; + for (int i = arg_len; i < co->co_nlocalsplus; i++) { - JitOptRef local = _Py_uop_sym_new_unknown(ctx); + JitOptRef local = default_null ? _Py_uop_sym_new_null(ctx) : _Py_uop_sym_new_unknown(ctx); frame->locals[i] = local; }