diff --git a/Grammar/python.gram b/Grammar/python.gram index 7ae00c6f005e7e..110136af81b596 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -1251,8 +1251,7 @@ invalid_expression: # !(NAME STRING) is not matched so we don't show this error with some invalid string prefixes like: kf"dsfsdf" # Soft keywords need to also be ignored because they can be parsed as NAME NAME | !(NAME STRING | SOFT_KEYWORD) a=disjunction b=expression_without_invalid { - _PyPegen_check_legacy_stmt(p, a) ? NULL : p->tokens[p->mark-1]->level == 0 ? NULL : - RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid syntax. Perhaps you forgot a comma?") } + _PyPegen_raise_error_for_missing_comma(p, a, b) } | a=disjunction 'if' b=disjunction !('else'|':') { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "expected 'else' after 'if' expression") } | a=disjunction 'if' b=disjunction 'else' !expression { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("expected expression after 'else', but statement is given") } diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 0dfd2a251bd6d5..35659de22b799d 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1366,7 +1366,7 @@ _PyOpcode_macro_expansion[256] = { [CALL_KW_BOUND_METHOD] = { .nuops = 6, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION_KW, 2, 1 }, { _EXPAND_METHOD_KW, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_KW_NON_PY] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE_KW, OPARG_SIMPLE, 3 }, { _CALL_KW_NON_PY, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_KW_PY] = { .nuops = 6, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION_KW, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [CALL_LEN] = { .nuops = 3, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_LEN, OPARG_SIMPLE, 3 }, { _CALL_LEN, OPARG_SIMPLE, 3 } } }, + [CALL_LEN] = { .nuops = 5, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_LEN, OPARG_SIMPLE, 3 }, { _CALL_LEN, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 } } }, [CALL_LIST_APPEND] = { .nuops = 4, .uops = { { _GUARD_CALLABLE_LIST_APPEND, OPARG_SIMPLE, 3 }, { _GUARD_NOS_NOT_NULL, OPARG_SIMPLE, 3 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 3 }, { _CALL_LIST_APPEND, OPARG_SIMPLE, 3 } } }, [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, @@ -1376,7 +1376,7 @@ _PyOpcode_macro_expansion[256] = { [CALL_PY_EXACT_ARGS] = { .nuops = 8, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_PY_GENERAL] = { .nuops = 6, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_STR_1] = { .nuops = 4, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_STR_1, OPARG_SIMPLE, 3 }, { _CALL_STR_1, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_TUPLE_1] = { .nuops = 4, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_TUPLE_1, OPARG_SIMPLE, 3 }, { _CALL_TUPLE_1, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_TUPLE_1] = { .nuops = 5, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_TUPLE_1, OPARG_SIMPLE, 3 }, { _CALL_TUPLE_1, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_TYPE_1] = { .nuops = 3, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_TYPE_1, OPARG_SIMPLE, 3 }, { _CALL_TYPE_1, OPARG_SIMPLE, 3 } } }, [CHECK_EG_MATCH] = { .nuops = 1, .uops = { { _CHECK_EG_MATCH, OPARG_SIMPLE, 0 } } }, [CHECK_EXC_MATCH] = { .nuops = 1, .uops = { { _CHECK_EXC_MATCH, OPARG_SIMPLE, 0 } } }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 553636cbb5914a..ce1ad5a4c8a5f6 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -431,7 +431,7 @@ extern "C" { #define _CALL_INTRINSIC_2_r21 624 #define _CALL_ISINSTANCE_r31 625 #define _CALL_KW_NON_PY_r11 626 -#define _CALL_LEN_r31 627 +#define _CALL_LEN_r33 627 #define _CALL_LIST_APPEND_r30 628 #define _CALL_METHOD_DESCRIPTOR_FAST_r01 629 #define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r01 630 @@ -439,7 +439,7 @@ extern "C" { #define _CALL_METHOD_DESCRIPTOR_O_r01 632 #define _CALL_NON_PY_GENERAL_r01 633 #define _CALL_STR_1_r31 634 -#define _CALL_TUPLE_1_r31 635 +#define _CALL_TUPLE_1_r32 635 #define _CALL_TYPE_1_r31 636 #define _CHECK_AND_ALLOCATE_OBJECT_r00 637 #define _CHECK_ATTR_CLASS_r01 638 diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 436defbb8df570..1e74588d3aa62e 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -2543,7 +2543,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, { -1, -1, -1 }, { -1, -1, -1 }, - { 1, 0, _CALL_TUPLE_1_r31 }, + { 2, 0, _CALL_TUPLE_1_r32 }, }, }, [_CHECK_AND_ALLOCATE_OBJECT] = { @@ -2624,7 +2624,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, { -1, -1, -1 }, { -1, -1, -1 }, - { 1, 0, _CALL_LEN_r31 }, + { 3, 0, _CALL_LEN_r33 }, }, }, [_GUARD_CALLABLE_ISINSTANCE] = { @@ -3739,7 +3739,7 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_GUARD_CALLABLE_TUPLE_1_r13] = _GUARD_CALLABLE_TUPLE_1, [_GUARD_CALLABLE_TUPLE_1_r23] = _GUARD_CALLABLE_TUPLE_1, [_GUARD_CALLABLE_TUPLE_1_r33] = _GUARD_CALLABLE_TUPLE_1, - [_CALL_TUPLE_1_r31] = _CALL_TUPLE_1, + [_CALL_TUPLE_1_r32] = _CALL_TUPLE_1, [_CHECK_AND_ALLOCATE_OBJECT_r00] = _CHECK_AND_ALLOCATE_OBJECT, [_CREATE_INIT_FRAME_r01] = _CREATE_INIT_FRAME, [_EXIT_INIT_CHECK_r10] = _EXIT_INIT_CHECK, @@ -3751,7 +3751,7 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_GUARD_CALLABLE_LEN_r13] = _GUARD_CALLABLE_LEN, [_GUARD_CALLABLE_LEN_r23] = _GUARD_CALLABLE_LEN, [_GUARD_CALLABLE_LEN_r33] = _GUARD_CALLABLE_LEN, - [_CALL_LEN_r31] = _CALL_LEN, + [_CALL_LEN_r33] = _CALL_LEN, [_GUARD_CALLABLE_ISINSTANCE_r03] = _GUARD_CALLABLE_ISINSTANCE, [_GUARD_CALLABLE_ISINSTANCE_r13] = _GUARD_CALLABLE_ISINSTANCE, [_GUARD_CALLABLE_ISINSTANCE_r23] = _GUARD_CALLABLE_ISINSTANCE, @@ -4042,7 +4042,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_CALL_KW_NON_PY] = "_CALL_KW_NON_PY", [_CALL_KW_NON_PY_r11] = "_CALL_KW_NON_PY_r11", [_CALL_LEN] = "_CALL_LEN", - [_CALL_LEN_r31] = "_CALL_LEN_r31", + [_CALL_LEN_r33] = "_CALL_LEN_r33", [_CALL_LIST_APPEND] = "_CALL_LIST_APPEND", [_CALL_LIST_APPEND_r30] = "_CALL_LIST_APPEND_r30", [_CALL_METHOD_DESCRIPTOR_FAST] = "_CALL_METHOD_DESCRIPTOR_FAST", @@ -4058,7 +4058,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_CALL_STR_1] = "_CALL_STR_1", [_CALL_STR_1_r31] = "_CALL_STR_1_r31", [_CALL_TUPLE_1] = "_CALL_TUPLE_1", - [_CALL_TUPLE_1_r31] = "_CALL_TUPLE_1_r31", + [_CALL_TUPLE_1_r32] = "_CALL_TUPLE_1_r32", [_CALL_TYPE_1] = "_CALL_TYPE_1", [_CALL_TYPE_1_r31] = "_CALL_TYPE_1_r31", [_CHECK_AND_ALLOCATE_OBJECT] = "_CHECK_AND_ALLOCATE_OBJECT", diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index cbff9694742490..46fef2048babe7 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -878,6 +878,12 @@ class MessageID(MsgID): class InvalidMessageID(MessageID): token_type = 'invalid-message-id' +class MessageIDList(TokenList): + token_type = 'message-id-list' + + @property + def message_ids(self): + return [x for x in self if x.token_type=='msg-id'] class Header(TokenList): token_type = 'header' @@ -2175,6 +2181,32 @@ def parse_message_id(value): return message_id +def parse_message_ids(value): + """in-reply-to = "In-Reply-To:" 1*msg-id CRLF + references = "References:" 1*msg-id CRLF + """ + message_id_list = MessageIDList() + while value: + if value[0] == ',': + # message id list separated with commas - this is invalid, + # but happens rather frequently in the wild + message_id_list.defects.append( + errors.InvalidHeaderDefect("comma in msg-id list")) + message_id_list.append( + WhiteSpaceTerminal(' ', 'invalid-comma-replacement')) + value = value[1:] + continue + try: + token, value = get_msg_id(value) + message_id_list.append(token) + except errors.HeaderParseError as ex: + token = get_unstructured(value) + message_id_list.append(InvalidMessageID(token)) + message_id_list.defects.append( + errors.InvalidHeaderDefect("Invalid msg-id: {!r}".format(ex))) + break + return message_id_list + # # XXX: As I begin to add additional header parsers, I'm realizing we probably # have two level of parser routines: the get_XXX methods that get a token in diff --git a/Lib/email/headerregistry.py b/Lib/email/headerregistry.py index 543141dc427ebe..0e8698efc0b966 100644 --- a/Lib/email/headerregistry.py +++ b/Lib/email/headerregistry.py @@ -534,6 +534,18 @@ def parse(cls, value, kwds): kwds['defects'].extend(parse_tree.all_defects) +class ReferencesHeader: + + max_count = 1 + value_parser = staticmethod(parser.parse_message_ids) + + @classmethod + def parse(cls, value, kwds): + kwds['parse_tree'] = parse_tree = cls.value_parser(value) + kwds['decoded'] = str(parse_tree) + kwds['defects'].extend(parse_tree.all_defects) + + # The header factory # _default_header_map = { @@ -557,6 +569,8 @@ def parse(cls, value, kwds): 'content-disposition': ContentDispositionHeader, 'content-transfer-encoding': ContentTransferEncodingHeader, 'message-id': MessageIDHeader, + 'in-reply-to': ReferencesHeader, + 'references': ReferencesHeader, } class HeaderRegistry: diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 2e7485e89fdac5..0e7fd62c28a065 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1925,6 +1925,21 @@ def testfunc(n): uops = get_opnames(ex) self.assertNotIn("_GUARD_IS_NOT_NONE_POP", uops) + def test_call_tuple_1_pop_top(self): + def testfunc(n): + x = 0 + for _ in range(n): + t = tuple(()) + x += len(t) == 0 + return x + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, TIER2_THRESHOLD) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_CALL_TUPLE_1", uops) + self.assertIn("_POP_TOP_NOP", uops) + def test_call_str_1(self): def testfunc(n): x = 0 @@ -2051,6 +2066,7 @@ def testfunc(n): self.assertIn("_CALL_LEN", uops) self.assertNotIn("_GUARD_NOS_INT", uops) self.assertNotIn("_GUARD_TOS_INT", uops) + self.assertIn("_POP_TOP_NOP", uops) def test_call_len_known_length_small_int(self): # Make sure that len(t) is optimized for a tuple of length 5. diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index f7f9f9c4e2fbb5..f33844910beee4 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -2867,6 +2867,81 @@ def test_get_msg_id_ws_only_local(self): ) self.assertEqual(msg_id.token_type, 'msg-id') + def test_parse_message_ids_valid(self): + message_ids = self._test_parse_x( + parser.parse_message_ids, + " ", + " ", + " ", + [], + ) + self.assertEqual(message_ids.token_type, 'message-id-list') + + def test_parse_message_ids_empty(self): + message_ids = self._test_parse_x( + parser.parse_message_ids, + " ", + " ", + " ", + [errors.InvalidHeaderDefect], + ) + self.assertEqual(message_ids.token_type, 'message-id-list') + + def test_parse_message_ids_comment(self): + message_ids = self._test_parse_x( + parser.parse_message_ids, + " (foo's message from \"bar\")", + " (foo's message from \"bar\")", + " ", + [], + ) + self.assertEqual(message_ids.message_ids[0].value, ' ') + self.assertEqual(message_ids.token_type, 'message-id-list') + + def test_parse_message_ids_no_sep(self): + message_ids = self._test_parse_x( + parser.parse_message_ids, + "", + "", + "", + [], + ) + self.assertEqual(message_ids.message_ids[0].value, '') + self.assertEqual(message_ids.message_ids[1].value, '') + self.assertEqual(message_ids.token_type, 'message-id-list') + + def test_parse_message_ids_comma_sep(self): + message_ids = self._test_parse_x( + parser.parse_message_ids, + ",", + " ", + " ", + [errors.InvalidHeaderDefect], + ) + self.assertEqual(message_ids.message_ids[0].value, '') + self.assertEqual(message_ids.message_ids[1].value, '') + self.assertEqual(message_ids.token_type, 'message-id-list') + + def test_parse_message_ids_invalid_id(self): + message_ids = self._test_parse_x( + parser.parse_message_ids, + "", + "", + "", + [errors.InvalidHeaderDefect]*2, + ) + self.assertEqual(message_ids.token_type, 'message-id-list') + + def test_parse_message_ids_broken_ang(self): + message_ids = self._test_parse_x( + parser.parse_message_ids, + " >bar@foo", + " >bar@foo", + " >bar@foo", + [errors.InvalidHeaderDefect]*1, + ) + self.assertEqual(message_ids.token_type, 'message-id-list') + @parameterize diff --git a/Lib/test/test_email/test_headerregistry.py b/Lib/test/test_email/test_headerregistry.py index 7138aa4c556d1f..95c6afbee41ef5 100644 --- a/Lib/test/test_email/test_headerregistry.py +++ b/Lib/test/test_email/test_headerregistry.py @@ -1821,5 +1821,18 @@ def test_message_id_header_is_not_folded(self): h.fold(policy=policy.default.clone(max_line_length=20)), 'Message-ID:\n <ईमेलfromMessage@wők.com>\n') + def test_fold_references(self): + h = self.make_header( + 'References', + ' ' + '' + ) + self.assertEqual( + h.fold(policy=policy.default.clone(max_line_length=20)), + 'References: ' + '\n' + ' \n') + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index ade8f273a1e088..93f0b98de71d81 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -3336,6 +3336,20 @@ def test_multiline_compiler_error_points_to_the_end(self): lineno=3 ) + def test_multiline_string_concat_missing_comma_points_to_last_string(self): + # gh-142236: For multi-line string concatenations with a missing comma, + # the error should point to the last string, not the first. + self._check_error( + "print(\n" + ' "line1"\n' + ' "line2"\n' + ' "line3"\n' + " x=1\n" + ")", + "Perhaps you forgot a comma", + lineno=4, # Points to "line3", the last string + ) + @support.cpython_only def test_syntax_error_on_deeply_nested_blocks(self): # This raises a SyntaxError, it used to raise a SystemError. Context diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-23-22-52-20.gh-issue-134584.qbiQfG.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-23-22-52-20.gh-issue-134584.qbiQfG.rst new file mode 100644 index 00000000000000..34df14750464ce --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-23-22-52-20.gh-issue-134584.qbiQfG.rst @@ -0,0 +1 @@ +Eliminate redundant refcounting from ``_CALL_TUPLE_1``. Patch by Noam Cohen diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-12-06-00-38-37.gh-issue-142236.m3EF9E.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-06-00-38-37.gh-issue-142236.m3EF9E.rst new file mode 100644 index 00000000000000..a8d37b49de7160 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-12-06-00-38-37.gh-issue-142236.m3EF9E.rst @@ -0,0 +1,3 @@ +Improve the "Perhaps you forgot a comma?" syntax error for multi-line string +concatenations to point to the last string instead of the first, making it +easier to locate where the comma is missing. Patch by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Library/2025-07-29-11-37-22.gh-issue-79986.fnJbE_.rst b/Misc/NEWS.d/next/Library/2025-07-29-11-37-22.gh-issue-79986.fnJbE_.rst new file mode 100644 index 00000000000000..327bbf869bce09 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-29-11-37-22.gh-issue-79986.fnJbE_.rst @@ -0,0 +1,3 @@ +Add parsing for ``References`` and ``In-Reply-To`` headers to the :mod:`email` +library that parses the header content as lists of message id tokens. This +prevents them from being folded incorrectly. diff --git a/Misc/NEWS.d/next/Library/2025-12-08-18-12-44.gh-issue-142438.UF_0nd.rst b/Misc/NEWS.d/next/Library/2025-12-08-18-12-44.gh-issue-142438.UF_0nd.rst new file mode 100644 index 00000000000000..ec6b3ff09e3a8c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-08-18-12-44.gh-issue-142438.UF_0nd.rst @@ -0,0 +1 @@ +Fixed a possible leaked GIL in _PySSL_keylog_callback. diff --git a/Modules/_ssl/debughelpers.c b/Modules/_ssl/debughelpers.c index aee446d0ccb1b8..866c172e4996f7 100644 --- a/Modules/_ssl/debughelpers.c +++ b/Modules/_ssl/debughelpers.c @@ -131,7 +131,7 @@ _PySSL_keylog_callback(const SSL *ssl, const char *line) PyThread_type_lock lock = get_state_sock(ssl_obj)->keylog_lock; assert(lock != NULL); if (ssl_obj->ctx->keylog_bio == NULL) { - return; + goto done; } /* * The lock is neither released on exit nor on fork(). The lock is @@ -155,6 +155,8 @@ _PySSL_keylog_callback(const SSL *ssl, const char *line) ssl_obj->ctx->keylog_filename); ssl_obj->exc = PyErr_GetRaisedException(); } + +done: PyGILState_Release(threadstate); } diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index fc609b2707c6c6..221cfc5a934198 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -7903,7 +7903,7 @@ py_posix_spawn(int use_posix_spawnp, PyObject *module, path_t *path, PyObject *a if (argc < 1) { PyErr_Format(PyExc_ValueError, "%s: argv must not be empty", func_name); - return NULL; + goto exit; } if (!PyMapping_Check(env) && env != Py_None) { diff --git a/PC/clinic/winreg.c.h b/PC/clinic/winreg.c.h index d76a8d8aef8cb8..92cf6e8a9be187 100644 --- a/PC/clinic/winreg.c.h +++ b/PC/clinic/winreg.c.h @@ -73,19 +73,13 @@ PyDoc_STRVAR(winreg_HKEYType___enter____doc__, #define WINREG_HKEYTYPE___ENTER___METHODDEF \ {"__enter__", (PyCFunction)winreg_HKEYType___enter__, METH_NOARGS, winreg_HKEYType___enter____doc__}, -static PyHKEYObject * +static PyObject * winreg_HKEYType___enter___impl(PyHKEYObject *self); static PyObject * winreg_HKEYType___enter__(PyObject *self, PyObject *Py_UNUSED(ignored)) { - PyObject *return_value = NULL; - PyHKEYObject *_return_value; - - _return_value = winreg_HKEYType___enter___impl((PyHKEYObject *)self); - return_value = (PyObject *)_return_value; - - return return_value; + return winreg_HKEYType___enter___impl((PyHKEYObject *)self); } #endif /* (defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) || defined(MS_WINDOWS_GAMES)) */ @@ -1842,4 +1836,4 @@ winreg_QueryReflectionKey(PyObject *module, PyObject *arg) #ifndef WINREG_QUERYREFLECTIONKEY_METHODDEF #define WINREG_QUERYREFLECTIONKEY_METHODDEF #endif /* !defined(WINREG_QUERYREFLECTIONKEY_METHODDEF) */ -/*[clinic end generated code: output=ce7e8e38884851fb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=97295995db2c24e9 input=a9049054013a1b77]*/ diff --git a/PC/winreg.c b/PC/winreg.c index bcb02b12299055..26bcd259efd987 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -264,19 +264,8 @@ class HKEY_return_converter(CReturnConverter): self.err_occurred_if_null_pointer("_return_value", data) data.return_conversion.append( 'return_value = PyHKEY_FromHKEY(_PyModule_GetState(module), _return_value);\n') - -# HACK: this only works for PyHKEYObjects, nothing else. -# Should this be generalized and enshrined in clinic.py, -# destroy this converter with prejudice. -class self_return_converter(CReturnConverter): - type = 'PyHKEYObject *' - - def render(self, function, data): - self.declare(data) - data.return_conversion.append( - 'return_value = (PyObject *)_return_value;\n') [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=4979f33998ffb6f8]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=b34c8217647f5fef]*/ #include "clinic/winreg.c.h" @@ -333,14 +322,14 @@ winreg_HKEYType_Detach_impl(PyHKEYObject *self) } /*[clinic input] -winreg.HKEYType.__enter__ -> self +winreg.HKEYType.__enter__ [clinic start generated code]*/ -static PyHKEYObject * +static PyObject * winreg_HKEYType___enter___impl(PyHKEYObject *self) -/*[clinic end generated code: output=52c34986dab28990 input=c40fab1f0690a8e2]*/ +/*[clinic end generated code: output=70ec10933068a08c input=85f6abf60774c88c]*/ { - return (PyHKEYObject*)Py_XNewRef(self); + return Py_XNewRef(self); } diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index b7a5b9d5e307b1..50856686335a14 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -947,6 +947,35 @@ _PyPegen_check_legacy_stmt(Parser *p, expr_ty name) { return 0; } +void * +_PyPegen_raise_error_for_missing_comma(Parser *p, expr_ty a, expr_ty b) +{ + // Don't raise for legacy statements like "print x" or "exec x" + if (_PyPegen_check_legacy_stmt(p, a)) { + return NULL; + } + // Only raise inside parentheses/brackets (level > 0) + if (p->tokens[p->mark - 1]->level == 0) { + return NULL; + } + // For multi-line expressions (like string concatenations), point to the + // last line instead of the first for a more helpful error message. + // Use a->col_offset as the starting column since all strings in the + // concatenation typically share the same indentation. + if (a->end_lineno > a->lineno) { + return RAISE_ERROR_KNOWN_LOCATION( + p, PyExc_SyntaxError, a->end_lineno, a->col_offset, + a->end_lineno, a->end_col_offset, + "invalid syntax. Perhaps you forgot a comma?" + ); + } + return RAISE_ERROR_KNOWN_LOCATION( + p, PyExc_SyntaxError, a->lineno, a->col_offset, + b->end_lineno, b->end_col_offset, + "invalid syntax. Perhaps you forgot a comma?" + ); +} + static ResultTokenWithMetadata * result_token_with_metadata(Parser *p, void *result, PyObject *metadata) { diff --git a/Parser/parser.c b/Parser/parser.c index 648b3702d8ff71..09bfb5725a2ec3 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -21445,7 +21445,7 @@ invalid_expression_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ invalid_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!(NAME STRING | SOFT_KEYWORD) disjunction expression_without_invalid")); - _res = _PyPegen_check_legacy_stmt ( p , a ) ? NULL : p -> tokens [p -> mark - 1] -> level == 0 ? NULL : RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , b , "invalid syntax. Perhaps you forgot a comma?" ); + _res = _PyPegen_raise_error_for_missing_comma ( p , a , b ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; diff --git a/Parser/pegen.h b/Parser/pegen.h index b8f887608b104e..be5333eb2684ae 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -358,6 +358,7 @@ expr_ty _PyPegen_ensure_real(Parser *p, expr_ty); asdl_seq *_PyPegen_join_sequences(Parser *, asdl_seq *, asdl_seq *); int _PyPegen_check_barry_as_flufl(Parser *, Token *); int _PyPegen_check_legacy_stmt(Parser *p, expr_ty t); +void *_PyPegen_raise_error_for_missing_comma(Parser *p, expr_ty a, expr_ty b); ResultTokenWithMetadata *_PyPegen_check_fstring_conversion(Parser *p, Token *, expr_ty t); ResultTokenWithMetadata *_PyPegen_setup_full_format_spec(Parser *, Token *, asdl_expr_seq *, int, int, int, int, PyArena *); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 9eac58af3e2622..cb39a23d951077 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -810,7 +810,7 @@ dummy_func( assert(next_instr->op.code == STORE_FAST); next_oparg = next_instr->op.arg; #else - next_oparg = (int)CURRENT_OPERAND0(); + next_oparg = (int)CURRENT_OPERAND0_16(); #endif _PyStackRef *target_local = &GETLOCAL(next_oparg); assert(PyUnicode_CheckExact(left_o)); @@ -4068,17 +4068,14 @@ dummy_func( DEOPT_IF(callable_o != (PyObject *)&PyTuple_Type); } - op(_CALL_TUPLE_1, (callable, null, arg -- res)) { + op(_CALL_TUPLE_1, (callable, null, arg -- res, a)) { PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); STAT_INC(CALL, hit); PyObject *res_o = PySequence_Tuple(arg_o); - DEAD(null); - DEAD(callable); - (void)callable; // Silence compiler warnings about unused variables - (void)null; - PyStackRef_CLOSE(arg); + a = arg; + INPUTS_DEAD(); ERROR_IF(res_o == NULL); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -4089,6 +4086,7 @@ dummy_func( _GUARD_NOS_NULL + _GUARD_CALLABLE_TUPLE_1 + _CALL_TUPLE_1 + + POP_TOP + _CHECK_PERIODIC_AT_END; op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { @@ -4282,7 +4280,9 @@ dummy_func( unused/2 + _GUARD_NOS_NULL + _GUARD_CALLABLE_LEN + - _CALL_LEN; + _CALL_LEN + + POP_TOP + + POP_TOP; op(_GUARD_CALLABLE_LEN, (callable, unused, unused -- callable, unused, unused)){ PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); @@ -4290,9 +4290,8 @@ dummy_func( DEOPT_IF(callable_o != interp->callable_cache.len); } - op(_CALL_LEN, (callable, null, arg -- res)) { + op(_CALL_LEN, (callable, null, arg -- res, a, c)) { /* len(o) */ - (void)null; STAT_INC(CALL, hit); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); Py_ssize_t len_i = PyObject_Length(arg_o); @@ -4304,9 +4303,9 @@ dummy_func( if (res_o == NULL) { ERROR_NO_POP(); } - PyStackRef_CLOSE(arg); - DEAD(null); - PyStackRef_CLOSE(callable); + a = arg; + c = callable; + INPUTS_DEAD(); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -5455,6 +5454,12 @@ dummy_func( } } + label(pop_3_error) { + stack_pointer -= 3; + assert(WITHIN_STACK_BOUNDS()); + goto error; + } + label(pop_2_error) { stack_pointer -= 2; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index d4ea1162eba444..26312d0ea5aa5c 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -450,8 +450,12 @@ do { \ } while (0) #define CURRENT_OPARG() (next_uop[-1].oparg) -#define CURRENT_OPERAND0() (next_uop[-1].operand0) -#define CURRENT_OPERAND1() (next_uop[-1].operand1) +#define CURRENT_OPERAND0_64() (next_uop[-1].operand0) +#define CURRENT_OPERAND1_64() (next_uop[-1].operand1) +#define CURRENT_OPERAND0_32() (next_uop[-1].operand0) +#define CURRENT_OPERAND1_32() (next_uop[-1].operand1) +#define CURRENT_OPERAND0_16() (next_uop[-1].operand0) +#define CURRENT_OPERAND1_16() (next_uop[-1].operand1) #define CURRENT_TARGET() (next_uop[-1].target) #define JUMP_TO_JUMP_TARGET() goto jump_to_jump_target diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index a64fd5b28f1d57..598fea4dfbf692 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4645,7 +4645,7 @@ assert(next_instr->op.code == STORE_FAST); next_oparg = next_instr->op.arg; #else - next_oparg = (int)CURRENT_OPERAND0(); + next_oparg = (int)CURRENT_OPERAND0_16(); #endif _PyStackRef *target_local = &GETLOCAL(next_oparg); assert(PyUnicode_CheckExact(left_o)); @@ -4692,7 +4692,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; right = _stack_item_1; left = _stack_item_0; - PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; @@ -4730,7 +4730,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; right = _stack_item_1; left = _stack_item_0; - PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); @@ -6736,7 +6736,7 @@ case _GUARD_GLOBALS_VERSION_r00: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); - uint16_t version = (uint16_t)CURRENT_OPERAND0(); + uint16_t version = (uint16_t)CURRENT_OPERAND0_16(); PyDictObject *dict = (PyDictObject *)GLOBALS(); if (!PyDict_CheckExact(dict)) { UOP_STAT_INC(uopcode, miss); @@ -6759,7 +6759,7 @@ CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef _stack_item_0 = _tos_cache0; - uint16_t version = (uint16_t)CURRENT_OPERAND0(); + uint16_t version = (uint16_t)CURRENT_OPERAND0_16(); PyDictObject *dict = (PyDictObject *)GLOBALS(); if (!PyDict_CheckExact(dict)) { UOP_STAT_INC(uopcode, miss); @@ -6784,7 +6784,7 @@ assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; - uint16_t version = (uint16_t)CURRENT_OPERAND0(); + uint16_t version = (uint16_t)CURRENT_OPERAND0_16(); PyDictObject *dict = (PyDictObject *)GLOBALS(); if (!PyDict_CheckExact(dict)) { UOP_STAT_INC(uopcode, miss); @@ -6811,7 +6811,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; - uint16_t version = (uint16_t)CURRENT_OPERAND0(); + uint16_t version = (uint16_t)CURRENT_OPERAND0_16(); PyDictObject *dict = (PyDictObject *)GLOBALS(); if (!PyDict_CheckExact(dict)) { UOP_STAT_INC(uopcode, miss); @@ -6837,8 +6837,8 @@ CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef res; - uint16_t version = (uint16_t)CURRENT_OPERAND0(); - uint16_t index = (uint16_t)CURRENT_OPERAND1(); + uint16_t version = (uint16_t)CURRENT_OPERAND0_16(); + uint16_t index = (uint16_t)CURRENT_OPERAND1_16(); PyDictObject *dict = (PyDictObject *)GLOBALS(); if (!PyDict_CheckExact(dict)) { UOP_STAT_INC(uopcode, miss); @@ -6883,8 +6883,8 @@ CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef res; - uint16_t version = (uint16_t)CURRENT_OPERAND0(); - uint16_t index = (uint16_t)CURRENT_OPERAND1(); + uint16_t version = (uint16_t)CURRENT_OPERAND0_16(); + uint16_t index = (uint16_t)CURRENT_OPERAND1_16(); PyDictObject *dict = (PyDictObject *)BUILTINS(); if (!PyDict_CheckExact(dict)) { UOP_STAT_INC(uopcode, miss); @@ -7936,7 +7936,7 @@ assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef owner; owner = stack_pointer[-1]; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { @@ -7958,7 +7958,7 @@ _PyStackRef owner; _PyStackRef _stack_item_0 = _tos_cache0; owner = _stack_item_0; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { @@ -7979,7 +7979,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; owner = _stack_item_1; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { @@ -8002,7 +8002,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; owner = _stack_item_2; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { @@ -8023,7 +8023,7 @@ assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef owner; owner = stack_pointer[-1]; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(type_version != 0); if (!LOCK_OBJECT(owner_o)) { @@ -8054,7 +8054,7 @@ _PyStackRef owner; _PyStackRef _stack_item_0 = _tos_cache0; owner = _stack_item_0; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(type_version != 0); if (!LOCK_OBJECT(owner_o)) { @@ -8084,7 +8084,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; owner = _stack_item_1; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(type_version != 0); if (!LOCK_OBJECT(owner_o)) { @@ -8116,7 +8116,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; owner = _stack_item_2; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(type_version != 0); if (!LOCK_OBJECT(owner_o)) { @@ -8235,7 +8235,7 @@ _PyStackRef attr; _PyStackRef _stack_item_0 = _tos_cache0; owner = _stack_item_0; - uint16_t offset = (uint16_t)CURRENT_OPERAND0(); + uint16_t offset = (uint16_t)CURRENT_OPERAND0_16(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); @@ -8280,8 +8280,8 @@ _PyStackRef attr; _PyStackRef _stack_item_0 = _tos_cache0; owner = _stack_item_0; - uint32_t dict_version = (uint32_t)CURRENT_OPERAND0(); - uint16_t index = (uint16_t)CURRENT_OPERAND1(); + uint32_t dict_version = (uint32_t)CURRENT_OPERAND0_32(); + uint16_t index = (uint16_t)CURRENT_OPERAND1_16(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) { UOP_STAT_INC(uopcode, miss); @@ -8342,7 +8342,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); owner = _stack_item_0; - uint16_t hint = (uint16_t)CURRENT_OPERAND0(); + uint16_t hint = (uint16_t)CURRENT_OPERAND0_16(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict = _PyObject_GetManagedDict(owner_o); @@ -8428,7 +8428,7 @@ _PyStackRef attr; _PyStackRef _stack_item_0 = _tos_cache0; owner = _stack_item_0; - uint16_t index = (uint16_t)CURRENT_OPERAND0(); + uint16_t index = (uint16_t)CURRENT_OPERAND0_16(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); PyObject **addr = (PyObject **)((char *)owner_o + index); PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); @@ -8472,7 +8472,7 @@ assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef owner; owner = stack_pointer[-1]; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); if (!PyType_Check(owner_o)) { UOP_STAT_INC(uopcode, miss); @@ -8499,7 +8499,7 @@ _PyStackRef owner; _PyStackRef _stack_item_0 = _tos_cache0; owner = _stack_item_0; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); if (!PyType_Check(owner_o)) { UOP_STAT_INC(uopcode, miss); @@ -8525,7 +8525,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; owner = _stack_item_1; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); if (!PyType_Check(owner_o)) { UOP_STAT_INC(uopcode, miss); @@ -8553,7 +8553,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; owner = _stack_item_2; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); if (!PyType_Check(owner_o)) { UOP_STAT_INC(uopcode, miss); @@ -8581,7 +8581,7 @@ _PyStackRef attr; _PyStackRef _stack_item_0 = _tos_cache0; owner = _stack_item_0; - PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); @@ -8612,7 +8612,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); owner = _stack_item_0; - PyObject *fget = (PyObject *)CURRENT_OPERAND0(); + PyObject *fget = (PyObject *)CURRENT_OPERAND0_64(); assert((oparg & 1) == 0); assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; @@ -8763,7 +8763,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; owner = _stack_item_1; value = _stack_item_0; - uint16_t offset = (uint16_t)CURRENT_OPERAND0(); + uint16_t offset = (uint16_t)CURRENT_OPERAND0_16(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); STAT_INC(STORE_ATTR, hit); assert(_PyObject_GetManagedDict(owner_o) == NULL); @@ -8798,7 +8798,7 @@ oparg = CURRENT_OPARG(); owner = _stack_item_1; value = _stack_item_0; - uint16_t hint = (uint16_t)CURRENT_OPERAND0(); + uint16_t hint = (uint16_t)CURRENT_OPERAND0_16(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictObject *dict = _PyObject_GetManagedDict(owner_o); @@ -8874,7 +8874,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; owner = _stack_item_1; value = _stack_item_0; - uint16_t index = (uint16_t)CURRENT_OPERAND0(); + uint16_t index = (uint16_t)CURRENT_OPERAND0_16(); PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); if (!LOCK_OBJECT(owner_o)) { UOP_STAT_INC(uopcode, miss); @@ -11133,7 +11133,7 @@ assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef owner; owner = stack_pointer[-1]; - uint32_t keys_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t keys_version = (uint32_t)CURRENT_OPERAND0_32(); PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; @@ -11156,7 +11156,7 @@ _PyStackRef owner; _PyStackRef _stack_item_0 = _tos_cache0; owner = _stack_item_0; - uint32_t keys_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t keys_version = (uint32_t)CURRENT_OPERAND0_32(); PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; @@ -11178,7 +11178,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; owner = _stack_item_1; - uint32_t keys_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t keys_version = (uint32_t)CURRENT_OPERAND0_32(); PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; @@ -11202,7 +11202,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; owner = _stack_item_2; - uint32_t keys_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t keys_version = (uint32_t)CURRENT_OPERAND0_32(); PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; @@ -11227,7 +11227,7 @@ _PyStackRef self; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); assert(oparg & 1); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -11252,7 +11252,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); owner = _stack_item_0; - PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); assert(oparg & 1); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -11276,7 +11276,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; oparg = CURRENT_OPARG(); owner = _stack_item_1; - PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); assert(oparg & 1); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -11299,7 +11299,7 @@ _PyStackRef self; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); assert(oparg & 1); assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); @@ -11325,7 +11325,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); owner = _stack_item_0; - PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); assert(oparg & 1); assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); @@ -11350,7 +11350,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; oparg = CURRENT_OPARG(); owner = _stack_item_1; - PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); assert(oparg & 1); assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); @@ -11374,7 +11374,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); owner = _stack_item_0; - PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); assert((oparg & 1) == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -11398,7 +11398,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); owner = _stack_item_0; - PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); assert((oparg & 1) == 0); assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); @@ -11420,7 +11420,7 @@ assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef owner; owner = stack_pointer[-1]; - uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0(); + uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0_16(); char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); if (dict != NULL) { @@ -11442,7 +11442,7 @@ _PyStackRef owner; _PyStackRef _stack_item_0 = _tos_cache0; owner = _stack_item_0; - uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0(); + uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0_16(); char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); if (dict != NULL) { @@ -11463,7 +11463,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; owner = _stack_item_1; - uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0(); + uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0_16(); char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); if (dict != NULL) { @@ -11486,7 +11486,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; owner = _stack_item_2; - uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0(); + uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0_16(); char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); if (dict != NULL) { @@ -11510,7 +11510,7 @@ _PyStackRef self; oparg = CURRENT_OPARG(); owner = stack_pointer[-1]; - PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); assert(oparg & 1); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -11535,7 +11535,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); owner = _stack_item_0; - PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); assert(oparg & 1); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -11559,7 +11559,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; oparg = CURRENT_OPARG(); owner = _stack_item_1; - PyObject *descr = (PyObject *)CURRENT_OPERAND0(); + PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); assert(oparg & 1); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -11656,7 +11656,7 @@ _PyStackRef callable; oparg = CURRENT_OPARG(); callable = stack_pointer[-2 - oparg]; - uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyFunction_Check(callable_o)) { UOP_STAT_INC(uopcode, miss); @@ -11677,8 +11677,8 @@ case _CHECK_FUNCTION_VERSION_INLINE_r00: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); - uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); - PyObject *callable_o = (PyObject *)CURRENT_OPERAND1(); + uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *callable_o = (PyObject *)CURRENT_OPERAND1_64(); assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { @@ -11695,8 +11695,8 @@ CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef _stack_item_0 = _tos_cache0; - uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); - PyObject *callable_o = (PyObject *)CURRENT_OPERAND1(); + uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *callable_o = (PyObject *)CURRENT_OPERAND1_64(); assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { @@ -11715,8 +11715,8 @@ assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; - uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); - PyObject *callable_o = (PyObject *)CURRENT_OPERAND1(); + uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *callable_o = (PyObject *)CURRENT_OPERAND1_64(); assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { @@ -11737,8 +11737,8 @@ _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; - uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); - PyObject *callable_o = (PyObject *)CURRENT_OPERAND1(); + uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); + PyObject *callable_o = (PyObject *)CURRENT_OPERAND1_64(); assert(PyFunction_Check(callable_o)); PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { @@ -11762,7 +11762,7 @@ oparg = CURRENT_OPARG(); null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (Py_TYPE(callable_o) != &PyMethod_Type) { UOP_STAT_INC(uopcode, miss); @@ -12888,47 +12888,42 @@ break; } - case _CALL_TUPLE_1_r31: { + case _CALL_TUPLE_1_r32: { CHECK_CURRENT_CACHED_VALUES(3); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef arg; - _PyStackRef null; - _PyStackRef callable; _PyStackRef res; + _PyStackRef a; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; oparg = CURRENT_OPARG(); arg = _stack_item_2; - null = _stack_item_1; - callable = _stack_item_0; PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); assert(oparg == 1); STAT_INC(CALL, hit); - stack_pointer[0] = callable; - stack_pointer[1] = null; + stack_pointer[0] = _stack_item_0; + stack_pointer[1] = _stack_item_1; stack_pointer[2] = arg; stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PySequence_Tuple(arg_o); stack_pointer = _PyFrame_GetStackPointer(frame); - (void)callable; - (void)null; - stack_pointer += -3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + a = arg; if (res_o == NULL) { + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } res = PyStackRef_FromPyObjectSteal(res_o); + _tos_cache1 = a; _tos_cache0 = res; - _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } @@ -12941,7 +12936,7 @@ oparg = CURRENT_OPARG(); self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - uint32_t type_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyStackRef_IsNull(self_or_null)) { UOP_STAT_INC(uopcode, miss); @@ -13375,24 +13370,23 @@ break; } - case _CALL_LEN_r31: { + case _CALL_LEN_r33: { CHECK_CURRENT_CACHED_VALUES(3); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef arg; - _PyStackRef null; _PyStackRef callable; _PyStackRef res; + _PyStackRef a; + _PyStackRef c; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; arg = _stack_item_2; - null = _stack_item_1; callable = _stack_item_0; - (void)null; STAT_INC(CALL, hit); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); stack_pointer[0] = callable; - stack_pointer[1] = null; + stack_pointer[1] = _stack_item_1; stack_pointer[2] = arg; stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -13409,21 +13403,15 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable); - stack_pointer = _PyFrame_GetStackPointer(frame); + a = arg; + c = callable; res = PyStackRef_FromPyObjectSteal(res_o); + _tos_cache2 = c; + _tos_cache1 = a; _tos_cache0 = res; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); break; } @@ -14123,7 +14111,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); callable = stack_pointer[-2 - oparg]; - uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (!PyFunction_Check(callable_o)) { UOP_STAT_INC(uopcode, miss); @@ -14151,7 +14139,7 @@ oparg = CURRENT_OPARG(); null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - uint32_t func_version = (uint32_t)CURRENT_OPERAND0(); + uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); if (Py_TYPE(callable_o) != &PyMethod_Type) { UOP_STAT_INC(uopcode, miss); @@ -15403,7 +15391,7 @@ case _SET_IP_r00: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); - PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0_64(); frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; SET_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); @@ -15414,7 +15402,7 @@ CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef _stack_item_0 = _tos_cache0; - PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0_64(); frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); @@ -15427,7 +15415,7 @@ assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; - PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0_64(); frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; @@ -15442,7 +15430,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; - PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0_64(); frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; _tos_cache2 = _stack_item_2; _tos_cache1 = _stack_item_1; @@ -15455,7 +15443,7 @@ case _CHECK_STACK_SPACE_OPERAND_r00: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); - uint32_t framesize = (uint32_t)CURRENT_OPERAND0(); + uint32_t framesize = (uint32_t)CURRENT_OPERAND0_32(); assert(framesize <= INT_MAX); if (!_PyThreadState_HasStackSpace(tstate, framesize)) { UOP_STAT_INC(uopcode, miss); @@ -15476,7 +15464,7 @@ CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef _stack_item_0 = _tos_cache0; - uint32_t framesize = (uint32_t)CURRENT_OPERAND0(); + uint32_t framesize = (uint32_t)CURRENT_OPERAND0_32(); assert(framesize <= INT_MAX); if (!_PyThreadState_HasStackSpace(tstate, framesize)) { UOP_STAT_INC(uopcode, miss); @@ -15499,7 +15487,7 @@ assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; - uint32_t framesize = (uint32_t)CURRENT_OPERAND0(); + uint32_t framesize = (uint32_t)CURRENT_OPERAND0_32(); assert(framesize <= INT_MAX); if (!_PyThreadState_HasStackSpace(tstate, framesize)) { UOP_STAT_INC(uopcode, miss); @@ -15524,7 +15512,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; - uint32_t framesize = (uint32_t)CURRENT_OPERAND0(); + uint32_t framesize = (uint32_t)CURRENT_OPERAND0_32(); assert(framesize <= INT_MAX); if (!_PyThreadState_HasStackSpace(tstate, framesize)) { UOP_STAT_INC(uopcode, miss); @@ -15619,7 +15607,7 @@ case _EXIT_TRACE_r00: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); - PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); _PyExitData *exit = (_PyExitData *)exit_p; #if defined(Py_DEBUG) && !defined(_Py_JIT) const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER) @@ -15646,7 +15634,7 @@ CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef _stack_item_0 = _tos_cache0; - PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); _PyExitData *exit = (_PyExitData *)exit_p; #if defined(Py_DEBUG) && !defined(_Py_JIT) const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER) @@ -15681,7 +15669,7 @@ assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; - PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); _PyExitData *exit = (_PyExitData *)exit_p; #if defined(Py_DEBUG) && !defined(_Py_JIT) const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER) @@ -15719,7 +15707,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; - PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); _PyExitData *exit = (_PyExitData *)exit_p; #if defined(Py_DEBUG) && !defined(_Py_JIT) const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER) @@ -15756,7 +15744,7 @@ case _DYNAMIC_EXIT_r00: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); - PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); #if defined(Py_DEBUG) && !defined(_Py_JIT) _PyExitData *exit = (_PyExitData *)exit_p; _Py_CODEUNIT *target = frame->instr_ptr; @@ -15780,7 +15768,7 @@ CHECK_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef _stack_item_0 = _tos_cache0; - PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); #if defined(Py_DEBUG) && !defined(_Py_JIT) _PyExitData *exit = (_PyExitData *)exit_p; _Py_CODEUNIT *target = frame->instr_ptr; @@ -15812,7 +15800,7 @@ assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; - PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); #if defined(Py_DEBUG) && !defined(_Py_JIT) _PyExitData *exit = (_PyExitData *)exit_p; _Py_CODEUNIT *target = frame->instr_ptr; @@ -15847,7 +15835,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; - PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); #if defined(Py_DEBUG) && !defined(_Py_JIT) _PyExitData *exit = (_PyExitData *)exit_p; _Py_CODEUNIT *target = frame->instr_ptr; @@ -15946,7 +15934,7 @@ CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); value = PyStackRef_FromPyObjectNew(ptr); _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); @@ -15959,7 +15947,7 @@ assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); value = PyStackRef_FromPyObjectNew(ptr); _tos_cache1 = value; _tos_cache0 = _stack_item_0; @@ -15974,7 +15962,7 @@ _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); value = PyStackRef_FromPyObjectNew(ptr); _tos_cache2 = value; _tos_cache1 = _stack_item_1; @@ -15991,7 +15979,7 @@ _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; pop = _stack_item_0; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -16008,7 +15996,7 @@ CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); value = PyStackRef_FromPyObjectBorrow(ptr); _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); @@ -16021,7 +16009,7 @@ assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); value = PyStackRef_FromPyObjectBorrow(ptr); _tos_cache1 = value; _tos_cache0 = _stack_item_0; @@ -16036,7 +16024,7 @@ _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); value = PyStackRef_FromPyObjectBorrow(ptr); _tos_cache2 = value; _tos_cache1 = _stack_item_1; @@ -16147,7 +16135,7 @@ _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; pop = _stack_item_0; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(pop); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -16170,7 +16158,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; pop2 = _stack_item_1; pop1 = _stack_item_0; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); stack_pointer[0] = pop1; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -16201,7 +16189,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; null = _stack_item_1; callable = _stack_item_0; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); (void)null; _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(callable); @@ -16228,7 +16216,7 @@ pop = _stack_item_2; null = _stack_item_1; callable = _stack_item_0; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); stack_pointer[0] = callable; stack_pointer[1] = null; stack_pointer += 2; @@ -16266,7 +16254,7 @@ pop1 = _stack_item_1; null = _stack_item_0; callable = stack_pointer[-1]; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); stack_pointer[0] = null; stack_pointer[1] = pop1; stack_pointer += 2; @@ -16301,7 +16289,7 @@ _PyStackRef value; _PyStackRef new; old = stack_pointer[-1]; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); new = old; value = PyStackRef_FromPyObjectNew(ptr); _tos_cache1 = new; @@ -16321,7 +16309,7 @@ _PyStackRef new; _PyStackRef _stack_item_0 = _tos_cache0; old = _stack_item_0; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); new = old; value = PyStackRef_FromPyObjectNew(ptr); _tos_cache1 = new; @@ -16340,7 +16328,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; old = _stack_item_1; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); new = old; value = PyStackRef_FromPyObjectNew(ptr); _tos_cache2 = new; @@ -16358,7 +16346,7 @@ _PyStackRef value; _PyStackRef new; old = stack_pointer[-1]; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); new = old; value = PyStackRef_FromPyObjectBorrow(ptr); _tos_cache1 = new; @@ -16378,7 +16366,7 @@ _PyStackRef new; _PyStackRef _stack_item_0 = _tos_cache0; old = _stack_item_0; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); new = old; value = PyStackRef_FromPyObjectBorrow(ptr); _tos_cache1 = new; @@ -16397,7 +16385,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; old = _stack_item_1; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0(); + PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); new = old; value = PyStackRef_FromPyObjectBorrow(ptr); _tos_cache2 = new; @@ -16411,7 +16399,7 @@ case _START_EXECUTOR_r00: { CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); - PyObject *executor = (PyObject *)CURRENT_OPERAND0(); + PyObject *executor = (PyObject *)CURRENT_OPERAND0_64(); #ifndef _Py_JIT assert(current_executor == (_PyExecutorObject*)executor); #endif @@ -16646,7 +16634,7 @@ CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); oparg = CURRENT_OPARG(); - uint32_t target = (uint32_t)CURRENT_OPERAND0(); + uint32_t target = (uint32_t)CURRENT_OPERAND0_32(); assert(oparg == 0); frame->instr_ptr = _PyFrame_GetBytecode(frame) + target; SET_CURRENT_CACHED_VALUES(0); @@ -16972,7 +16960,7 @@ CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); #define OFFSET_OF__PUSH_FRAME ((0)) - PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF__PUSH_FRAME; if (target != (_Py_CODEUNIT *)ip) { frame->instr_ptr += OFFSET_OF__PUSH_FRAME; @@ -16993,7 +16981,7 @@ assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef _stack_item_0 = _tos_cache0; #define OFFSET_OF__PUSH_FRAME ((0)) - PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF__PUSH_FRAME; if (target != (_Py_CODEUNIT *)ip) { frame->instr_ptr += OFFSET_OF__PUSH_FRAME; @@ -17016,7 +17004,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; #define OFFSET_OF__PUSH_FRAME ((0)) - PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF__PUSH_FRAME; if (target != (_Py_CODEUNIT *)ip) { frame->instr_ptr += OFFSET_OF__PUSH_FRAME; @@ -17041,7 +17029,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; #define OFFSET_OF__PUSH_FRAME ((0)) - PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF__PUSH_FRAME; if (target != (_Py_CODEUNIT *)ip) { frame->instr_ptr += OFFSET_OF__PUSH_FRAME; @@ -17064,7 +17052,7 @@ CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); #define OFFSET_OF_YIELD_VALUE ((1+INLINE_CACHE_ENTRIES_SEND)) - PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_YIELD_VALUE; if (target != (_Py_CODEUNIT *)ip) { frame->instr_ptr += OFFSET_OF_YIELD_VALUE; @@ -17085,7 +17073,7 @@ assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef _stack_item_0 = _tos_cache0; #define OFFSET_OF_YIELD_VALUE ((1+INLINE_CACHE_ENTRIES_SEND)) - PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_YIELD_VALUE; if (target != (_Py_CODEUNIT *)ip) { frame->instr_ptr += OFFSET_OF_YIELD_VALUE; @@ -17108,7 +17096,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; #define OFFSET_OF_YIELD_VALUE ((1+INLINE_CACHE_ENTRIES_SEND)) - PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_YIELD_VALUE; if (target != (_Py_CODEUNIT *)ip) { frame->instr_ptr += OFFSET_OF_YIELD_VALUE; @@ -17133,7 +17121,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; #define OFFSET_OF_YIELD_VALUE ((1+INLINE_CACHE_ENTRIES_SEND)) - PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_YIELD_VALUE; if (target != (_Py_CODEUNIT *)ip) { frame->instr_ptr += OFFSET_OF_YIELD_VALUE; @@ -17156,7 +17144,7 @@ CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); #define OFFSET_OF_RETURN_VALUE ((frame->return_offset)) - PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_VALUE; if (target != (_Py_CODEUNIT *)ip) { frame->instr_ptr += OFFSET_OF_RETURN_VALUE; @@ -17177,7 +17165,7 @@ assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef _stack_item_0 = _tos_cache0; #define OFFSET_OF_RETURN_VALUE ((frame->return_offset)) - PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_VALUE; if (target != (_Py_CODEUNIT *)ip) { frame->instr_ptr += OFFSET_OF_RETURN_VALUE; @@ -17200,7 +17188,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; #define OFFSET_OF_RETURN_VALUE ((frame->return_offset)) - PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_VALUE; if (target != (_Py_CODEUNIT *)ip) { frame->instr_ptr += OFFSET_OF_RETURN_VALUE; @@ -17225,7 +17213,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; #define OFFSET_OF_RETURN_VALUE ((frame->return_offset)) - PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_VALUE; if (target != (_Py_CODEUNIT *)ip) { frame->instr_ptr += OFFSET_OF_RETURN_VALUE; @@ -17248,7 +17236,7 @@ CHECK_CURRENT_CACHED_VALUES(0); assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); #define OFFSET_OF_RETURN_GENERATOR ((frame->return_offset)) - PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_GENERATOR; if (target != (_Py_CODEUNIT *)ip) { frame->instr_ptr += OFFSET_OF_RETURN_GENERATOR; @@ -17269,7 +17257,7 @@ assert(WITHIN_STACK_BOUNDS_WITH_CACHE()); _PyStackRef _stack_item_0 = _tos_cache0; #define OFFSET_OF_RETURN_GENERATOR ((frame->return_offset)) - PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_GENERATOR; if (target != (_Py_CODEUNIT *)ip) { frame->instr_ptr += OFFSET_OF_RETURN_GENERATOR; @@ -17292,7 +17280,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; #define OFFSET_OF_RETURN_GENERATOR ((frame->return_offset)) - PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_GENERATOR; if (target != (_Py_CODEUNIT *)ip) { frame->instr_ptr += OFFSET_OF_RETURN_GENERATOR; @@ -17317,7 +17305,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; #define OFFSET_OF_RETURN_GENERATOR ((frame->return_offset)) - PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_GENERATOR; if (target != (_Py_CODEUNIT *)ip) { frame->instr_ptr += OFFSET_OF_RETURN_GENERATOR; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 420cdcaed03339..a0e6fd2d7cba27 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -371,7 +371,7 @@ assert(next_instr->op.code == STORE_FAST); next_oparg = next_instr->op.arg; #else - next_oparg = (int)CURRENT_OPERAND0(); + next_oparg = (int)CURRENT_OPERAND0_16(); #endif _PyStackRef *target_local = &GETLOCAL(next_oparg); assert(PyUnicode_CheckExact(left_o)); @@ -3144,6 +3144,9 @@ _PyStackRef callable; _PyStackRef arg; _PyStackRef res; + _PyStackRef a; + _PyStackRef c; + _PyStackRef value; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _GUARD_NOS_NULL @@ -3169,7 +3172,6 @@ // _CALL_LEN { arg = stack_pointer[-1]; - (void)null; STAT_INC(CALL, hit); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -3183,21 +3185,30 @@ if (res_o == NULL) { JUMP_TO_LABEL(error); } + a = arg; + c = callable; + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _POP_TOP + { + value = c; + stack_pointer[-3] = res; + stack_pointer[-2] = a; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(arg); + PyStackRef_XCLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; + } + // _POP_TOP + { + value = a; + stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable); + PyStackRef_XCLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectSteal(res_o); } - stack_pointer[0] = res; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -4029,6 +4040,8 @@ _PyStackRef callable; _PyStackRef arg; _PyStackRef res; + _PyStackRef a; + _PyStackRef value; /* Skip 1 cache entry */ /* Skip 2 cache entries */ // _GUARD_NOS_NULL @@ -4059,23 +4072,24 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PySequence_Tuple(arg_o); stack_pointer = _PyFrame_GetStackPointer(frame); - (void)callable; - (void)null; - stack_pointer += -3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + a = arg; if (res_o == NULL) { - JUMP_TO_LABEL(error); + JUMP_TO_LABEL(pop_3_error); } res = PyStackRef_FromPyObjectSteal(res_o); } - // _CHECK_PERIODIC_AT_END + // _POP_TOP { - stack_pointer[0] = res; - stack_pointer += 1; + value = a; + stack_pointer[-3] = res; + stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _CHECK_PERIODIC_AT_END + { _PyFrame_SetStackPointer(frame, stack_pointer); int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -11849,6 +11863,13 @@ JUMP_TO_LABEL(error); #endif /* _Py_TAIL_CALL_INTERP */ /* BEGIN LABELS */ + LABEL(pop_3_error) + { + stack_pointer -= 3; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + LABEL(pop_2_error) { stack_pointer -= 2; diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index b2fa7d01e8f6c2..1a8a4392187eca 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -522,6 +522,7 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256]; static py_tail_call_funcptr instruction_funcptr_tracing_table[256]; +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_3_error(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 864ae17525e6b2..9e767464500d45 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -1097,7 +1097,7 @@ dummy_func(void) { } } - op(_CALL_TUPLE_1, (callable, null, arg -- res)) { + op(_CALL_TUPLE_1, (callable, null, arg -- res, a)) { if (sym_matches_type(arg, &PyTuple_Type)) { // e.g. tuple((1, 2)) or tuple(foo) where foo is known to be a tuple // Note: we must strip the reference information because it goes @@ -1107,6 +1107,7 @@ dummy_func(void) { else { res = sym_new_type(ctx, &PyTuple_Type); } + a = arg; } op(_GUARD_TOS_LIST, (tos -- tos)) { @@ -1201,7 +1202,7 @@ dummy_func(void) { sym_set_const(callable, (PyObject *)&PyUnicode_Type); } - op(_CALL_LEN, (callable, null, arg -- res)) { + op(_CALL_LEN, (callable, null, arg -- res, a, c)) { res = sym_new_type(ctx, &PyLong_Type); Py_ssize_t tuple_length = sym_tuple_length(arg); if (tuple_length >= 0) { @@ -1216,6 +1217,8 @@ dummy_func(void) { res = sym_new_const(ctx, temp); Py_DECREF(temp); } + a = arg; + c = callable; } op(_GET_LEN, (obj -- obj, len)) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 20192ebc593054..c3f122215ed7bd 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -2882,6 +2882,7 @@ case _CALL_TUPLE_1: { JitOptRef arg; JitOptRef res; + JitOptRef a; arg = stack_pointer[-1]; if (sym_matches_type(arg, &PyTuple_Type)) { res = PyJitRef_StripReferenceInfo(arg); @@ -2889,9 +2890,11 @@ else { res = sym_new_type(ctx, &PyTuple_Type); } - CHECK_STACK_BOUNDS(-2); + a = arg; + CHECK_STACK_BOUNDS(-1); stack_pointer[-3] = res; - stack_pointer += -2; + stack_pointer[-2] = a; + stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); break; } @@ -2999,8 +3002,12 @@ case _CALL_LEN: { JitOptRef arg; + JitOptRef callable; JitOptRef res; + JitOptRef a; + JitOptRef c; arg = stack_pointer[-1]; + callable = stack_pointer[-3]; res = sym_new_type(ctx, &PyLong_Type); Py_ssize_t tuple_length = sym_tuple_length(arg); if (tuple_length >= 0) { @@ -3020,10 +3027,11 @@ Py_DECREF(temp); stack_pointer += 2; } - CHECK_STACK_BOUNDS(-2); + a = arg; + c = callable; stack_pointer[-3] = res; - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + stack_pointer[-2] = a; + stack_pointer[-1] = c; break; } diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index 7e50a6ebafbfd3..9c86382cdc5610 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -222,12 +222,13 @@ def write_uop(uop: Uop, emitter: Emitter, stack: Stack, offset_strs: dict[str, t idx = 0 for cache in uop.caches: if cache.name != "unused": + bits = cache.size*16 if cache.size == 4: type = cast = "PyObject *" else: - type = f"uint{cache.size*16}_t " - cast = f"uint{cache.size*16}_t" - emitter.emit(f"{type}{cache.name} = ({cast})CURRENT_OPERAND{idx}();\n") + type = f"uint{bits}_t " + cast = f"uint{bits}_t" + emitter.emit(f"{type}{cache.name} = ({cast})CURRENT_OPERAND{idx}_{bits}();\n") idx += 1 reachable, storage = emitter.emit_tokens(uop, storage, None, False) if reachable: diff --git a/Tools/jit/_optimizers.py b/Tools/jit/_optimizers.py index 297b9517f6a27a..08d4e700652deb 100644 --- a/Tools/jit/_optimizers.py +++ b/Tools/jit/_optimizers.py @@ -96,6 +96,8 @@ class InstructionKind(enum.Enum): LONG_BRANCH = enum.auto() SHORT_BRANCH = enum.auto() RETURN = enum.auto() + SMALL_CONST_1 = enum.auto() + SMALL_CONST_2 = enum.auto() OTHER = enum.auto() @@ -172,6 +174,7 @@ class Optimizer: ) # Override everything that follows in subclasses: _supports_external_relocations = True + supports_small_constants = False _branches: typing.ClassVar[dict[str, tuple[str | None, str | None]]] = {} # Short branches are instructions that can branch within a micro-op, # but might not have the reach to branch anywhere within a trace. @@ -184,6 +187,9 @@ class Optimizer: _re_return: typing.ClassVar[re.Pattern[str]] = _RE_NEVER_MATCH text: str = "" globals: set[str] = dataclasses.field(default_factory=set) + _re_small_const_1 = _RE_NEVER_MATCH + _re_small_const_2 = _RE_NEVER_MATCH + const_reloc = "" def __post_init__(self) -> None: # Split the code into a linked list of basic blocks. A basic block is an @@ -253,6 +259,14 @@ def _parse_instruction(self, line: str) -> Instruction: elif match := self._re_return.match(line): name = line kind = InstructionKind.RETURN + elif match := self._re_small_const_1.match(line): + target = match["value"] + name = match["instruction"] + kind = InstructionKind.SMALL_CONST_1 + elif match := self._re_small_const_2.match(line): + target = match["value"] + name = match["instruction"] + kind = InstructionKind.SMALL_CONST_2 else: name, *_ = line.split(" ") kind = InstructionKind.OTHER @@ -385,7 +399,7 @@ def _remove_redundant_jumps(self) -> None: block.fallthrough = True block.instructions.pop() # Before: - # br ? FOO: + # branch FOO: # ... # FOO: # jump BAR @@ -461,6 +475,70 @@ def _fixup_external_labels(self) -> None: ) block.instructions.append(branch.update_target("0")) + def _make_temp_label(self, index: int) -> Instruction: + marker = f"jit_temp_{index}:" + return Instruction(InstructionKind.OTHER, "", marker, None) + + def _fixup_constants(self) -> None: + if not self.supports_small_constants: + return + index = 0 + for block in self._blocks(): + fixed: list[Instruction] = [] + small_const_index = -1 + for inst in block.instructions: + if inst.kind == InstructionKind.SMALL_CONST_1: + marker = f"jit_pending_{inst.target}{index}:" + fixed.append(self._make_temp_label(index)) + index += 1 + small_const_index = len(fixed) + fixed.append(inst) + elif inst.kind == InstructionKind.SMALL_CONST_2: + if small_const_index < 0: + fixed.append(inst) + continue + small_const_1 = fixed[small_const_index] + if not self._small_consts_match(small_const_1, inst): + small_const_index = -1 + fixed.append(inst) + continue + assert small_const_1.target is not None + if small_const_1.target.endswith("16"): + fixed[small_const_index] = self._make_temp_label(index) + index += 1 + else: + assert small_const_1.target.endswith("32") + patch_kind, replacement = self._small_const_1(small_const_1) + if replacement is not None: + label = f"{self.const_reloc}{patch_kind}_JIT_RELOCATION_CONST{small_const_1.target[:-3]}_JIT_RELOCATION_{index}:" + index += 1 + fixed[small_const_index - 1] = Instruction( + InstructionKind.OTHER, "", label, None + ) + fixed[small_const_index] = replacement + patch_kind, replacement = self._small_const_2(inst) + if replacement is not None: + assert inst.target is not None + label = f"{self.const_reloc}{patch_kind}_JIT_RELOCATION_CONST{inst.target[:-3]}_JIT_RELOCATION_{index}:" + index += 1 + fixed.append( + Instruction(InstructionKind.OTHER, "", label, None) + ) + fixed.append(replacement) + small_const_index = -1 + else: + fixed.append(inst) + block.instructions = fixed + + def _small_const_1(self, inst: Instruction) -> tuple[str, Instruction | None]: + raise NotImplementedError() + + def _small_const_2(self, inst: Instruction) -> tuple[str, Instruction | None]: + raise NotImplementedError() + + def _small_consts_match(self, inst1: Instruction, inst2: Instruction) -> bool: + raise NotImplementedError() + def run(self) -> None: """Run this optimizer.""" self._insert_continue_label() @@ -472,6 +550,7 @@ def run(self) -> None: self._remove_redundant_jumps() self._remove_unreachable() self._fixup_external_labels() + self._fixup_constants() self.path.write_text(self._body()) @@ -492,6 +571,54 @@ class OptimizerAArch64(Optimizer): # pylint: disable = too-few-public-methods # https://developer.arm.com/documentation/ddi0602/2025-09/Base-Instructions/RET--Return-from-subroutine- _re_return = re.compile(r"\s*ret\b") + supports_small_constants = True + _re_small_const_1 = re.compile( + r"\s*(?Padrp)\s+.*(?P_JIT_OP(ARG|ERAND(0|1))_(16|32)).*" + ) + _re_small_const_2 = re.compile( + r"\s*(?Pldr)\s+.*(?P_JIT_OP(ARG|ERAND(0|1))_(16|32)).*" + ) + const_reloc = "CUSTOM_AARCH64_CONST" + + def _get_reg(self, inst: Instruction) -> str: + _, rest = inst.text.split(inst.name) + reg, *_ = rest.split(",") + return reg.strip() + + def _small_const_1(self, inst: Instruction) -> tuple[str, Instruction | None]: + assert inst.kind is InstructionKind.SMALL_CONST_1 + assert inst.target is not None + if "16" in inst.target: + return "", None + pre, _ = inst.text.split(inst.name) + return "16a", Instruction( + InstructionKind.OTHER, "movz", f"{pre}movz {self._get_reg(inst)}, 0", None + ) + + def _small_const_2(self, inst: Instruction) -> tuple[str, Instruction | None]: + assert inst.kind is InstructionKind.SMALL_CONST_2 + assert inst.target is not None + pre, _ = inst.text.split(inst.name) + if "16" in inst.target: + return "16a", Instruction( + InstructionKind.OTHER, + "movz", + f"{pre}movz {self._get_reg(inst)}, 0", + None, + ) + else: + return "16b", Instruction( + InstructionKind.OTHER, + "movk", + f"{pre}movk {self._get_reg(inst)}, 0, lsl #16", + None, + ) + + def _small_consts_match(self, inst1: Instruction, inst2: Instruction) -> bool: + reg1 = self._get_reg(inst1) + reg2 = self._get_reg(inst2) + return reg1 == reg2 + class OptimizerX86(Optimizer): # pylint: disable = too-few-public-methods """i686-pc-windows-msvc/x86_64-apple-darwin/x86_64-unknown-linux-gnu""" diff --git a/Tools/jit/_schema.py b/Tools/jit/_schema.py index 4e86abe604972e..964e8bdbdc1010 100644 --- a/Tools/jit/_schema.py +++ b/Tools/jit/_schema.py @@ -9,6 +9,9 @@ "ARM64_RELOC_PAGE21", "ARM64_RELOC_PAGEOFF12", "ARM64_RELOC_UNSIGNED", + "CUSTOM_AARCH64_BRANCH19", + "CUSTOM_AARCH64_CONST_16", + "CUSTOM_AARCH64_CONST_32", "IMAGE_REL_AMD64_REL32", "IMAGE_REL_ARM64_BRANCH19", "IMAGE_REL_ARM64_BRANCH26", diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py index cdffd953ef9838..2b78d8013af5db 100644 --- a/Tools/jit/_stencils.py +++ b/Tools/jit/_stencils.py @@ -32,6 +32,12 @@ class HoleValue(enum.Enum): # The current uop's operand0 on 32-bit platforms (exposed as _JIT_OPERAND0_HI/LO): OPERAND0_HI = enum.auto() OPERAND0_LO = enum.auto() + # 16 and 32 bit versions of OPARG, OPERAND0 and OPERAND1 + OPARG_16 = enum.auto() + OPERAND0_16 = enum.auto() + OPERAND1_16 = enum.auto() + OPERAND0_32 = enum.auto() + OPERAND1_32 = enum.auto() # The current uop's operand1 on 64-bit platforms (exposed as _JIT_OPERAND1): OPERAND1 = enum.auto() # The current uop's operand1 on 32-bit platforms (exposed as _JIT_OPERAND1_HI/LO): @@ -59,6 +65,8 @@ class HoleValue(enum.Enum): "ARM64_RELOC_PAGEOFF12": "patch_aarch64_12", "ARM64_RELOC_UNSIGNED": "patch_64", "CUSTOM_AARCH64_BRANCH19": "patch_aarch64_19r", + "CUSTOM_AARCH64_CONST16a": "patch_aarch64_16a", + "CUSTOM_AARCH64_CONST16b": "patch_aarch64_16b", # x86_64-pc-windows-msvc: "IMAGE_REL_AMD64_REL32": "patch_x86_64_32rx", # aarch64-pc-windows-msvc: @@ -95,6 +103,7 @@ class HoleValue(enum.Enum): "X86_64_RELOC_SIGNED": "patch_32r", "X86_64_RELOC_UNSIGNED": "patch_64", } + # Translate HoleValues to C expressions: _HOLE_EXPRS = { HoleValue.CODE: "(uintptr_t)code", @@ -103,10 +112,15 @@ class HoleValue(enum.Enum): HoleValue.GOT: "", # These should all have been turned into DATA values by process_relocations: HoleValue.OPARG: "instruction->oparg", + HoleValue.OPARG_16: "instruction->oparg", HoleValue.OPERAND0: "instruction->operand0", + HoleValue.OPERAND0_16: "instruction->operand0", + HoleValue.OPERAND0_32: "instruction->operand0", HoleValue.OPERAND0_HI: "(instruction->operand0 >> 32)", HoleValue.OPERAND0_LO: "(instruction->operand0 & UINT32_MAX)", HoleValue.OPERAND1: "instruction->operand1", + HoleValue.OPERAND1_16: "instruction->operand1", + HoleValue.OPERAND1_32: "instruction->operand1", HoleValue.OPERAND1_HI: "(instruction->operand1 >> 32)", HoleValue.OPERAND1_LO: "(instruction->operand1 & UINT32_MAX)", HoleValue.TARGET: "instruction->target", @@ -201,7 +215,10 @@ def as_c(self, where: str) -> str: if self.symbol: if value: value += " + " - value += f"(uintptr_t)&{self.symbol}" + if self.symbol.startswith("CONST"): + value += f"instruction->{self.symbol[10:].lower()}" + else: + value += f"(uintptr_t)&{self.symbol}" if _signed(self.addend) or not value: if value: value += " + " diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index f92f3eac13bde5..5895e91c3c44ce 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -138,6 +138,7 @@ async def _compile( f"--target={self.triple}", "-DPy_BUILD_CORE_MODULE", "-D_DEBUG" if self.debug else "-DNDEBUG", + f"-DSUPPORTS_SMALL_CONSTS={1 if self.optimizer.supports_small_constants else 0}", f"-D_JIT_OPCODE={opname}", "-D_PyJIT_ACTIVE", "-D_Py_JIT", diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 93067977791c42..064b401bc3aca4 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -34,14 +34,38 @@ #include "jit.h" + +#undef CURRENT_OPERAND0_64 +#define CURRENT_OPERAND0_64() (_operand0_64) + +#undef CURRENT_OPERAND1_64 +#define CURRENT_OPERAND1_64() (_operand1_64) + + #undef CURRENT_OPARG +#undef CURRENT_OPERAND0_16 +#undef CURRENT_OPERAND0_32 +#undef CURRENT_OPERAND1_16 +#undef CURRENT_OPERAND1_32 + +#if SUPPORTS_SMALL_CONSTS + +#define CURRENT_OPARG() (_oparg_16) +#define CURRENT_OPERAND0_32() (_operand0_32) +#define CURRENT_OPERAND0_16() (_operand0_16) +#define CURRENT_OPERAND1_32() (_operand1_32) +#define CURRENT_OPERAND1_16() (_operand1_16) + +#else + #define CURRENT_OPARG() (_oparg) +#define CURRENT_OPERAND0_32() (_operand0_64) +#define CURRENT_OPERAND0_16() (_operand0_64) +#define CURRENT_OPERAND1_32() (_operand1_64) +#define CURRENT_OPERAND1_16() (_operand1_64) -#undef CURRENT_OPERAND0 -#define CURRENT_OPERAND0() (_operand0) +#endif -#undef CURRENT_OPERAND1 -#define CURRENT_OPERAND1() (_operand1) #undef CURRENT_TARGET #define CURRENT_TARGET() (_target) @@ -105,18 +129,26 @@ _JIT_ENTRY( int uopcode = _JIT_OPCODE; _Py_CODEUNIT *next_instr; // Other stuff we need handy: - PATCH_VALUE(uint16_t, _oparg, _JIT_OPARG) #if SIZEOF_VOID_P == 8 - PATCH_VALUE(uint64_t, _operand0, _JIT_OPERAND0) - PATCH_VALUE(uint64_t, _operand1, _JIT_OPERAND1) + PATCH_VALUE(uint64_t, _operand0_64, _JIT_OPERAND0) + PATCH_VALUE(uint64_t, _operand1_64, _JIT_OPERAND1) #else assert(SIZEOF_VOID_P == 4); PATCH_VALUE(uint32_t, _operand0_hi, _JIT_OPERAND0_HI) PATCH_VALUE(uint32_t, _operand0_lo, _JIT_OPERAND0_LO) - uint64_t _operand0 = ((uint64_t)_operand0_hi << 32) | _operand0_lo; + uint64_t _operand0_64 = ((uint64_t)_operand0_hi << 32) | _operand0_lo; PATCH_VALUE(uint32_t, _operand1_hi, _JIT_OPERAND1_HI) PATCH_VALUE(uint32_t, _operand1_lo, _JIT_OPERAND1_LO) - uint64_t _operand1 = ((uint64_t)_operand1_hi << 32) | _operand1_lo; + uint64_t _operand1_64 = ((uint64_t)_operand1_hi << 32) | _operand1_lo; +#endif +#if SUPPORTS_SMALL_CONSTS + PATCH_VALUE(uint32_t, _operand0_32, _JIT_OPERAND0_32) + PATCH_VALUE(uint32_t, _operand1_32, _JIT_OPERAND1_32) + PATCH_VALUE(uint16_t, _operand0_16, _JIT_OPERAND0_16) + PATCH_VALUE(uint16_t, _operand1_16, _JIT_OPERAND1_16) + PATCH_VALUE(uint16_t, _oparg_16, _JIT_OPARG_16) +#else + PATCH_VALUE(uint16_t, _oparg, _JIT_OPARG) #endif PATCH_VALUE(uint32_t, _target, _JIT_TARGET) OPT_STAT_INC(uops_executed);