From da4f71b415ab9fb06d8358089e59855a6ea43c28 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sun, 1 Oct 2023 15:24:21 -0700 Subject: [PATCH 1/4] Attempt to auto-generate the target for DEOPT_IF() --- Python/bytecodes.c | 352 ++++++++++++------------ Python/executor_cases.c.h | 73 +++-- Python/generated_cases.c.h | 202 +++++++++----- Tools/cases_generator/analysis.py | 2 +- Tools/cases_generator/generate_cases.py | 14 +- Tools/cases_generator/instructions.py | 14 +- Tools/cases_generator/stacking.py | 12 +- 7 files changed, 364 insertions(+), 305 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index f7681bd234a43f..035629dda6e7db 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -41,8 +41,6 @@ #include "ceval_macros.h" /* Flow control macros */ -#define DEOPT_IF(cond, instname) ((void)0) -#define ERROR_IF(cond, labelname) ((void)0) #define GO_TO_INSTRUCTION(instname) ((void)0) #define inst(name, ...) case name: @@ -153,13 +151,12 @@ dummy_func( inst(RESUME_CHECK, (--)) { #if defined(__EMSCRIPTEN__) - DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); + DEOPT_IF(_Py_emscripten_signal_clock == 0); _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif /* Possibly combine these two checks */ - DEOPT_IF(_PyFrame_GetCode(frame)->_co_instrumentation_version - != tstate->interp->monitoring_version, RESUME); - DEOPT_IF(_Py_atomic_load_relaxed_int32(&tstate->interp->ceval.eval_breaker), RESUME); + DEOPT_IF(_PyFrame_GetCode(frame)->_co_instrumentation_version != tstate->interp->monitoring_version); + DEOPT_IF(_Py_atomic_load_relaxed_int32(&tstate->interp->ceval.eval_breaker)); } inst(INSTRUMENTED_RESUME, (--)) { @@ -325,12 +322,12 @@ dummy_func( } inst(TO_BOOL_BOOL, (unused/1, unused/2, value -- value)) { - DEOPT_IF(!PyBool_Check(value), TO_BOOL); + DEOPT_IF(!PyBool_Check(value)); STAT_INC(TO_BOOL, hit); } inst(TO_BOOL_INT, (unused/1, unused/2, value -- res)) { - DEOPT_IF(!PyLong_CheckExact(value), TO_BOOL); + DEOPT_IF(!PyLong_CheckExact(value)); STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value)) { assert(_Py_IsImmortal(value)); @@ -343,7 +340,7 @@ dummy_func( } inst(TO_BOOL_LIST, (unused/1, unused/2, value -- res)) { - DEOPT_IF(!PyList_CheckExact(value), TO_BOOL); + DEOPT_IF(!PyList_CheckExact(value)); STAT_INC(TO_BOOL, hit); res = Py_SIZE(value) ? Py_True : Py_False; DECREF_INPUTS(); @@ -351,13 +348,13 @@ dummy_func( inst(TO_BOOL_NONE, (unused/1, unused/2, value -- res)) { // This one is a bit weird, because we expect *some* failures: - DEOPT_IF(!Py_IsNone(value), TO_BOOL); + DEOPT_IF(!Py_IsNone(value)); STAT_INC(TO_BOOL, hit); res = Py_False; } inst(TO_BOOL_STR, (unused/1, unused/2, value -- res)) { - DEOPT_IF(!PyUnicode_CheckExact(value), TO_BOOL); + DEOPT_IF(!PyUnicode_CheckExact(value)); STAT_INC(TO_BOOL, hit); if (value == &_Py_STR(empty)) { assert(_Py_IsImmortal(value)); @@ -373,7 +370,7 @@ dummy_func( inst(TO_BOOL_ALWAYS_TRUE, (unused/1, version/2, value -- res)) { // This one is a bit weird, because we expect *some* failures: assert(version); - DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL); + DEOPT_IF(Py_TYPE(value)->tp_version_tag != version); STAT_INC(TO_BOOL, hit); DECREF_INPUTS(); res = Py_True; @@ -397,8 +394,8 @@ dummy_func( }; op(_GUARD_BOTH_INT, (left, right -- left, right)) { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left)); + DEOPT_IF(!PyLong_CheckExact(right)); } op(_BINARY_OP_MULTIPLY_INT, (unused/1, left, right -- res)) { @@ -433,8 +430,8 @@ dummy_func( _GUARD_BOTH_INT + _BINARY_OP_SUBTRACT_INT; op(_GUARD_BOTH_FLOAT, (left, right -- left, right)) { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left)); + DEOPT_IF(!PyFloat_CheckExact(right)); } op(_BINARY_OP_MULTIPLY_FLOAT, (unused/1, left, right -- res)) { @@ -469,8 +466,8 @@ dummy_func( _GUARD_BOTH_FLOAT + _BINARY_OP_SUBTRACT_FLOAT; op(_GUARD_BOTH_UNICODE, (left, right -- left, right)) { - DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(left)); + DEOPT_IF(!PyUnicode_CheckExact(right)); } op(_BINARY_OP_ADD_UNICODE, (unused/1, left, right -- res)) { @@ -494,7 +491,7 @@ dummy_func( _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; assert(true_next.op.code == STORE_FAST); PyObject **target_local = &GETLOCAL(true_next.op.arg); - DEOPT_IF(*target_local != left, BINARY_OP); + DEOPT_IF(*target_local != left); STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. * @@ -574,13 +571,13 @@ dummy_func( } inst(BINARY_SUBSCR_LIST_INT, (unused/1, list, sub -- res)) { - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub)); + DEOPT_IF(!PyList_CheckExact(list)); // Deopt unless 0 <= sub < PyList_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); + DEOPT_IF(index >= PyList_GET_SIZE(list)); STAT_INC(BINARY_SUBSCR, hit); res = PyList_GET_ITEM(list, index); assert(res != NULL); @@ -590,14 +587,14 @@ dummy_func( } inst(BINARY_SUBSCR_STR_INT, (unused/1, str, sub -- res)) { - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyUnicode_CheckExact(str), BINARY_SUBSCR); - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub)); + DEOPT_IF(!PyUnicode_CheckExact(str)); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index, BINARY_SUBSCR); + DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index); // Specialize for reading an ASCII character from any string: Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c, BINARY_SUBSCR); + DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c); STAT_INC(BINARY_SUBSCR, hit); res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); @@ -605,13 +602,13 @@ dummy_func( } inst(BINARY_SUBSCR_TUPLE_INT, (unused/1, tuple, sub -- res)) { - DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR); - DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub)); + DEOPT_IF(!PyTuple_CheckExact(tuple)); // Deopt unless 0 <= sub < PyTuple_Size(list) - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR); + DEOPT_IF(index >= PyTuple_GET_SIZE(tuple)); STAT_INC(BINARY_SUBSCR, hit); res = PyTuple_GET_ITEM(tuple, index); assert(res != NULL); @@ -621,7 +618,7 @@ dummy_func( } inst(BINARY_SUBSCR_DICT, (unused/1, dict, sub -- res)) { - DEOPT_IF(!PyDict_CheckExact(dict), BINARY_SUBSCR); + DEOPT_IF(!PyDict_CheckExact(dict)); STAT_INC(BINARY_SUBSCR, hit); res = PyDict_GetItemWithError(dict, sub); if (res == NULL) { @@ -636,19 +633,19 @@ dummy_func( } inst(BINARY_SUBSCR_GETITEM, (unused/1, container, sub -- unused)) { - DEOPT_IF(tstate->interp->eval_frame, BINARY_SUBSCR); + DEOPT_IF(tstate->interp->eval_frame); PyTypeObject *tp = Py_TYPE(container); - DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE), BINARY_SUBSCR); + DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *cached = ht->_spec_cache.getitem; - DEOPT_IF(cached == NULL, BINARY_SUBSCR); + DEOPT_IF(cached == NULL); assert(PyFunction_Check(cached)); PyFunctionObject *getitem = (PyFunctionObject *)cached; uint32_t cached_version = ht->_spec_cache.getitem_version; - DEOPT_IF(getitem->func_version != cached_version, BINARY_SUBSCR); + DEOPT_IF(getitem->func_version != cached_version); PyCodeObject *code = (PyCodeObject *)getitem->func_code; assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize)); STAT_INC(BINARY_SUBSCR, hit); Py_INCREF(getitem); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2); @@ -693,14 +690,14 @@ dummy_func( } inst(STORE_SUBSCR_LIST_INT, (unused/1, value, list, sub -- )) { - DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR); - DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR); + DEOPT_IF(!PyLong_CheckExact(sub)); + DEOPT_IF(!PyList_CheckExact(list)); // Ensure nonnegative, zero-or-one-digit ints. - DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), STORE_SUBSCR); + DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; // Ensure index < len(list) - DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR); + DEOPT_IF(index >= PyList_GET_SIZE(list)); STAT_INC(STORE_SUBSCR, hit); PyObject *old_value = PyList_GET_ITEM(list, index); @@ -712,7 +709,7 @@ dummy_func( } inst(STORE_SUBSCR_DICT, (unused/1, value, dict, sub -- )) { - DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR); + DEOPT_IF(!PyDict_CheckExact(dict)); STAT_INC(STORE_SUBSCR, hit); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value); Py_DECREF(dict); @@ -1009,11 +1006,10 @@ dummy_func( } inst(SEND_GEN, (unused/1, receiver, v -- receiver, unused)) { - DEOPT_IF(tstate->interp->eval_frame, SEND); + DEOPT_IF(tstate->interp->eval_frame); PyGenObject *gen = (PyGenObject *)receiver; - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && - Py_TYPE(gen) != &PyCoro_Type, SEND); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING); STAT_INC(SEND, hit); _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; STACK_SHRINK(1); @@ -1194,8 +1190,8 @@ dummy_func( } inst(UNPACK_SEQUENCE_TWO_TUPLE, (unused/1, seq -- values[oparg])) { - DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq) != 2, UNPACK_SEQUENCE); + DEOPT_IF(!PyTuple_CheckExact(seq)); + DEOPT_IF(PyTuple_GET_SIZE(seq) != 2); assert(oparg == 2); STAT_INC(UNPACK_SEQUENCE, hit); values[0] = Py_NewRef(PyTuple_GET_ITEM(seq, 1)); @@ -1204,8 +1200,8 @@ dummy_func( } inst(UNPACK_SEQUENCE_TUPLE, (unused/1, seq -- values[oparg])) { - DEOPT_IF(!PyTuple_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); + DEOPT_IF(!PyTuple_CheckExact(seq)); + DEOPT_IF(PyTuple_GET_SIZE(seq) != oparg); STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyTuple_ITEMS(seq); for (int i = oparg; --i >= 0; ) { @@ -1215,8 +1211,8 @@ dummy_func( } inst(UNPACK_SEQUENCE_LIST, (unused/1, seq -- values[oparg])) { - DEOPT_IF(!PyList_CheckExact(seq), UNPACK_SEQUENCE); - DEOPT_IF(PyList_GET_SIZE(seq) != oparg, UNPACK_SEQUENCE); + DEOPT_IF(!PyList_CheckExact(seq)); + DEOPT_IF(PyList_GET_SIZE(seq) != oparg); STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyList_ITEMS(seq); for (int i = oparg; --i >= 0; ) { @@ -1412,15 +1408,15 @@ dummy_func( op(_GUARD_GLOBALS_VERSION, (version/1 --)) { PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict)); + DEOPT_IF(dict->ma_keys->dk_version != version); assert(DK_IS_UNICODE(dict->ma_keys)); } op(_GUARD_BUILTINS_VERSION, (version/1 --)) { PyDictObject *dict = (PyDictObject *)BUILTINS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict)); + DEOPT_IF(dict->ma_keys->dk_version != version); assert(DK_IS_UNICODE(dict->ma_keys)); } @@ -1428,7 +1424,7 @@ dummy_func( PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); res = entries[index].me_value; - DEOPT_IF(res == NULL, LOAD_GLOBAL); + DEOPT_IF(res == NULL); Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; @@ -1438,7 +1434,7 @@ dummy_func( PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); res = entries[index].me_value; - DEOPT_IF(res == NULL, LOAD_GLOBAL); + DEOPT_IF(res == NULL); Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; @@ -1763,8 +1759,8 @@ dummy_func( inst(LOAD_SUPER_ATTR_ATTR, (unused/1, global_super, class, self -- attr, unused if (0))) { assert(!(oparg & 1)); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type); + DEOPT_IF(!PyType_Check(class)); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); @@ -1774,8 +1770,8 @@ dummy_func( inst(LOAD_SUPER_ATTR_METHOD, (unused/1, global_super, class, self -- attr, self_or_null)) { assert(oparg & 1); - DEOPT_IF(global_super != (PyObject *)&PySuper_Type, LOAD_SUPER_ATTR); - DEOPT_IF(!PyType_Check(class), LOAD_SUPER_ATTR); + DEOPT_IF(global_super != (PyObject *)&PySuper_Type); + DEOPT_IF(!PyType_Check(class)); STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); PyTypeObject *cls = (PyTypeObject *)class; @@ -1864,22 +1860,20 @@ dummy_func( op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) { PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version); } op(_CHECK_MANAGED_OBJECT_HAS_VALUES, (owner -- owner)) { assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && - !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), - LOAD_ATTR); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)); } op(_LOAD_ATTR_INSTANCE_VALUE, (index/1, owner -- attr, null if (oparg & 1))) { PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); attr = _PyDictOrValues_GetValues(dorv)->values[index]; - DEOPT_IF(attr == NULL, LOAD_ATTR); + DEOPT_IF(attr == NULL); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1894,15 +1888,15 @@ dummy_func( unused/5; // Skip over rest of cache inst(LOAD_ATTR_MODULE, (unused/1, type_version/2, index/1, unused/5, owner -- attr, null if (oparg & 1))) { - DEOPT_IF(!PyModule_CheckExact(owner), LOAD_ATTR); + DEOPT_IF(!PyModule_CheckExact(owner)); PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner)->md_dict; assert(dict != NULL); - DEOPT_IF(dict->ma_keys->dk_version != type_version, LOAD_ATTR); + DEOPT_IF(dict->ma_keys->dk_version != type_version); assert(dict->ma_keys->dk_kind == DICT_KEYS_UNICODE); assert(index < dict->ma_keys->dk_nentries); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + index; attr = ep->me_value; - DEOPT_IF(attr == NULL, LOAD_ATTR); + DEOPT_IF(attr == NULL); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1912,27 +1906,27 @@ dummy_func( inst(LOAD_ATTR_WITH_HINT, (unused/1, type_version/2, index/1, unused/5, owner -- attr, null if (oparg & 1))) { PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv), LOAD_ATTR); + DEOPT_IF(_PyDictOrValues_IsValues(dorv)); PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(dict == NULL, LOAD_ATTR); + DEOPT_IF(dict == NULL); assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); uint16_t hint = index; - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, LOAD_ATTR); + DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries); if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, LOAD_ATTR); + DEOPT_IF(ep->me_key != name); attr = ep->me_value; } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, LOAD_ATTR); + DEOPT_IF(ep->me_key != name); attr = ep->me_value; } - DEOPT_IF(attr == NULL, LOAD_ATTR); + DEOPT_IF(attr == NULL); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1942,7 +1936,7 @@ dummy_func( op(_LOAD_ATTR_SLOT, (index/1, owner -- attr, null if (oparg & 1))) { char *addr = (char *)owner + index; attr = *(PyObject **)addr; - DEOPT_IF(attr == NULL, LOAD_ATTR); + DEOPT_IF(attr == NULL); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1957,9 +1951,8 @@ dummy_func( inst(LOAD_ATTR_CLASS, (unused/1, type_version/2, unused/2, descr/4, owner -- attr, null if (oparg & 1))) { - DEOPT_IF(!PyType_Check(owner), LOAD_ATTR); - DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, - LOAD_ATTR); + DEOPT_IF(!PyType_Check(owner)); + DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version); assert(type_version != 0); STAT_INC(LOAD_ATTR, hit); @@ -1972,18 +1965,18 @@ dummy_func( inst(LOAD_ATTR_PROPERTY, (unused/1, type_version/2, func_version/2, fget/4, owner -- unused, unused if (0))) { assert((oparg & 1) == 0); - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + DEOPT_IF(tstate->interp->eval_frame); PyTypeObject *cls = Py_TYPE(owner); - DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(cls->tp_version_tag != type_version); assert(type_version != 0); assert(Py_IS_TYPE(fget, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)fget; assert(func_version != 0); - DEOPT_IF(f->func_version != func_version, LOAD_ATTR); + DEOPT_IF(f->func_version != func_version); PyCodeObject *code = (PyCodeObject *)f->func_code; assert(code->co_argcount == 1); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize)); STAT_INC(LOAD_ATTR, hit); Py_INCREF(fget); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 1); @@ -1997,17 +1990,17 @@ dummy_func( inst(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, (unused/1, type_version/2, func_version/2, getattribute/4, owner -- unused, unused if (0))) { assert((oparg & 1) == 0); - DEOPT_IF(tstate->interp->eval_frame, LOAD_ATTR); + DEOPT_IF(tstate->interp->eval_frame); PyTypeObject *cls = Py_TYPE(owner); - DEOPT_IF(cls->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(cls->tp_version_tag != type_version); assert(type_version != 0); assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)getattribute; assert(func_version != 0); - DEOPT_IF(f->func_version != func_version, LOAD_ATTR); + DEOPT_IF(f->func_version != func_version); PyCodeObject *code = (PyCodeObject *)f->func_code; assert(code->co_argcount == 2); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize)); STAT_INC(LOAD_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); @@ -2025,7 +2018,7 @@ dummy_func( op(_GUARD_DORV_VALUES, (owner -- owner)) { assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); + DEOPT_IF(!_PyDictOrValues_IsValues(dorv)); } op(_STORE_ATTR_INSTANCE_VALUE, (index/1, value, owner --)) { @@ -2052,30 +2045,30 @@ dummy_func( inst(STORE_ATTR_WITH_HINT, (unused/1, type_version/2, hint/1, value, owner --)) { PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version); assert(tp->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(_PyDictOrValues_IsValues(dorv), STORE_ATTR); + DEOPT_IF(_PyDictOrValues_IsValues(dorv)); PyDictObject *dict = (PyDictObject *)_PyDictOrValues_GetDict(dorv); - DEOPT_IF(dict == NULL, STORE_ATTR); + DEOPT_IF(dict == NULL); assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries, STORE_ATTR); + DEOPT_IF(hint >= (size_t)dict->ma_keys->dk_nentries); PyObject *old_value; uint64_t new_version; if (DK_IS_UNICODE(dict->ma_keys)) { PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, STORE_ATTR); + DEOPT_IF(ep->me_key != name); old_value = ep->me_value; - DEOPT_IF(old_value == NULL, STORE_ATTR); + DEOPT_IF(old_value == NULL); new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); ep->me_value = value; } else { PyDictKeyEntry *ep = DK_ENTRIES(dict->ma_keys) + hint; - DEOPT_IF(ep->me_key != name, STORE_ATTR); + DEOPT_IF(ep->me_key != name); old_value = ep->me_value; - DEOPT_IF(old_value == NULL, STORE_ATTR); + DEOPT_IF(old_value == NULL); new_version = _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, value); ep->me_value = value; } @@ -2093,7 +2086,7 @@ dummy_func( op(_GUARD_TYPE_VERSION_STORE, (type_version/2, owner -- owner)) { PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version); } op(_STORE_ATTR_SLOT, (index/1, value, owner --)) { @@ -2140,8 +2133,8 @@ dummy_func( } inst(COMPARE_OP_FLOAT, (unused/1, left, right -- res)) { - DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyFloat_CheckExact(right), COMPARE_OP); + DEOPT_IF(!PyFloat_CheckExact(left)); + DEOPT_IF(!PyFloat_CheckExact(right)); STAT_INC(COMPARE_OP, hit); double dleft = PyFloat_AS_DOUBLE(left); double dright = PyFloat_AS_DOUBLE(right); @@ -2155,10 +2148,10 @@ dummy_func( // Similar to COMPARE_OP_FLOAT inst(COMPARE_OP_INT, (unused/1, left, right -- res)) { - DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyLong_CheckExact(right), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left), COMPARE_OP); - DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right), COMPARE_OP); + DEOPT_IF(!PyLong_CheckExact(left)); + DEOPT_IF(!PyLong_CheckExact(right)); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)left)); + DEOPT_IF(!_PyLong_IsCompact((PyLongObject *)right)); STAT_INC(COMPARE_OP, hit); assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && _PyLong_DigitCount((PyLongObject *)right) <= 1); @@ -2174,8 +2167,8 @@ dummy_func( // Similar to COMPARE_OP_FLOAT, but for ==, != only inst(COMPARE_OP_STR, (unused/1, left, right -- res)) { - DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), COMPARE_OP); + DEOPT_IF(!PyUnicode_CheckExact(left)); + DEOPT_IF(!PyUnicode_CheckExact(right)); STAT_INC(COMPARE_OP, hit); int eq = _PyUnicode_Equal(left, right); assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); @@ -2492,7 +2485,7 @@ dummy_func( } op(_ITER_CHECK_LIST, (iter -- iter)) { - DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type); } op(_ITER_JUMP_LIST, (iter -- iter)) { @@ -2548,7 +2541,7 @@ dummy_func( _ITER_NEXT_LIST; op(_ITER_CHECK_TUPLE, (iter -- iter)) { - DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type); } op(_ITER_JUMP_TUPLE, (iter -- iter)) { @@ -2605,7 +2598,7 @@ dummy_func( op(_ITER_CHECK_RANGE, (iter -- iter)) { _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type); } op(_ITER_JUMP_RANGE, (iter -- iter)) { @@ -2647,10 +2640,10 @@ dummy_func( _ITER_NEXT_RANGE; inst(FOR_ITER_GEN, (unused/1, iter -- iter, unused)) { - DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); + DEOPT_IF(tstate->interp->eval_frame); PyGenObject *gen = (PyGenObject *)iter; - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type, FOR_ITER); - DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, FOR_ITER); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type); + DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING); STAT_INC(FOR_ITER, hit); _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; _PyFrame_StackPush(gen_frame, Py_None); @@ -2790,16 +2783,13 @@ dummy_func( op(_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, (owner -- owner)) { assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && - !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), - LOAD_ATTR); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)); } op(_GUARD_KEYS_VERSION, (keys_version/2, owner -- owner)) { PyTypeObject *owner_cls = Py_TYPE(owner); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != - keys_version, LOAD_ATTR); + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version); } op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self if (1))) { @@ -2839,15 +2829,12 @@ dummy_func( assert((oparg & 1) == 0); PyTypeObject *owner_cls = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(owner_cls->tp_version_tag != type_version); assert(owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && - !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), - LOAD_ATTR); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv)); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != - keys_version, LOAD_ATTR); + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); DECREF_INPUTS(); @@ -2858,7 +2845,7 @@ dummy_func( assert((oparg & 1) == 0); PyTypeObject *owner_cls = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(owner_cls->tp_version_tag != type_version); assert(owner_cls->tp_dictoffset == 0); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); @@ -2869,12 +2856,12 @@ dummy_func( inst(LOAD_ATTR_METHOD_LAZY_DICT, (unused/1, type_version/2, unused/2, descr/4, owner -- attr, self if (1))) { assert(oparg & 1); PyTypeObject *owner_cls = Py_TYPE(owner); - DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(owner_cls->tp_version_tag != type_version); Py_ssize_t dictoffset = owner_cls->tp_dictoffset; assert(dictoffset > 0); PyObject *dict = *(PyObject **)((char *)owner + dictoffset); /* This object has a __dict__, just not yet created */ - DEOPT_IF(dict != NULL, LOAD_ATTR); + DEOPT_IF(dict != NULL); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); @@ -3002,8 +2989,8 @@ dummy_func( } op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); + DEOPT_IF(null != NULL); + DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type); } op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, unused, unused[oparg] -- func, self, unused[oparg])) { @@ -3016,22 +3003,22 @@ dummy_func( } op(_CHECK_PEP_523, (--)) { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame); } op(_CHECK_FUNCTION_EXACT_ARGS, (func_version/2, callable, self_or_null, unused[oparg] -- callable, self_or_null, unused[oparg])) { - DEOPT_IF(!PyFunction_Check(callable), CALL); + DEOPT_IF(!PyFunction_Check(callable)); PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); + DEOPT_IF(func->func_version != func_version); PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); + DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL)); } op(_CHECK_STACK_SPACE, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { PyFunctionObject *func = (PyFunctionObject *)callable; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize)); + DEOPT_IF(tstate->py_recursion_remaining <= 1); } op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _PyInterpreterFrame*)) { @@ -3094,24 +3081,24 @@ dummy_func( _PUSH_FRAME; inst(CALL_PY_WITH_DEFAULTS, (unused/1, func_version/2, callable, self_or_null, args[oparg] -- unused)) { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame); int argcount = oparg; if (self_or_null != NULL) { args--; argcount++; } - DEOPT_IF(!PyFunction_Check(callable), CALL); + DEOPT_IF(!PyFunction_Check(callable)); PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); + DEOPT_IF(func->func_version != func_version); PyCodeObject *code = (PyCodeObject *)func->func_code; assert(func->func_defaults); assert(PyTuple_CheckExact(func->func_defaults)); int defcount = (int)PyTuple_GET_SIZE(func->func_defaults); assert(defcount <= code->co_argcount); int min_args = code->co_argcount - defcount; - DEOPT_IF(argcount > code->co_argcount, CALL); - DEOPT_IF(argcount < min_args, CALL); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(argcount > code->co_argcount); + DEOPT_IF(argcount < min_args); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize)); STAT_INC(CALL, hit); _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, code->co_argcount); for (int i = 0; i < argcount; i++) { @@ -3130,9 +3117,9 @@ dummy_func( inst(CALL_TYPE_1, (unused/1, unused/2, callable, null, args[oparg] -- res)) { assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); + DEOPT_IF(null != NULL); PyObject *obj = args[0]; - DEOPT_IF(callable != (PyObject *)&PyType_Type, CALL); + DEOPT_IF(callable != (PyObject *)&PyType_Type); STAT_INC(CALL, hit); res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); @@ -3141,8 +3128,8 @@ dummy_func( inst(CALL_STR_1, (unused/1, unused/2, callable, null, args[oparg] -- res)) { assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyUnicode_Type, CALL); + DEOPT_IF(null != NULL); + DEOPT_IF(callable != (PyObject *)&PyUnicode_Type); STAT_INC(CALL, hit); PyObject *arg = args[0]; res = PyObject_Str(arg); @@ -3154,8 +3141,8 @@ dummy_func( inst(CALL_TUPLE_1, (unused/1, unused/2, callable, null, args[oparg] -- res)) { assert(oparg == 1); - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(callable != (PyObject *)&PyTuple_Type, CALL); + DEOPT_IF(null != NULL); + DEOPT_IF(callable != (PyObject *)&PyTuple_Type); STAT_INC(CALL, hit); PyObject *arg = args[0]; res = PySequence_Tuple(arg); @@ -3172,15 +3159,15 @@ dummy_func( * 3. Pushes the frame for ``__init__`` to the frame stack * */ _PyCallCache *cache = (_PyCallCache *)next_instr; - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(!PyType_Check(callable), CALL); + DEOPT_IF(null != NULL); + DEOPT_IF(!PyType_Check(callable)); PyTypeObject *tp = (PyTypeObject *)callable; - DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version), CALL); + DEOPT_IF(tp->tp_version_tag != read_u32(cache->func_version)); PyHeapTypeObject *cls = (PyHeapTypeObject *)callable; PyFunctionObject *init = (PyFunctionObject *)cls->_spec_cache.init; PyCodeObject *code = (PyCodeObject *)init->func_code; - DEOPT_IF(code->co_argcount != oparg+1, CALL); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); + DEOPT_IF(code->co_argcount != oparg+1); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)); STAT_INC(CALL, hit); PyObject *self = _PyType_NewManagedObject(tp); if (self == NULL) { @@ -3233,9 +3220,9 @@ dummy_func( args--; total_args++; } - DEOPT_IF(!PyType_Check(callable), CALL); + DEOPT_IF(!PyType_Check(callable)); PyTypeObject *tp = (PyTypeObject *)callable; - DEOPT_IF(tp->tp_vectorcall == NULL, CALL); + DEOPT_IF(tp->tp_vectorcall == NULL); STAT_INC(CALL, hit); res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); /* Free the arguments. */ @@ -3254,9 +3241,9 @@ dummy_func( args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O, CALL); + DEOPT_IF(total_args != 1); + DEOPT_IF(!PyCFunction_CheckExact(callable)); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_O); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); // This is slower but CPython promises to check all non-vectorcall @@ -3282,8 +3269,8 @@ dummy_func( args--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL, CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable)); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != METH_FASTCALL); STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); /* res = func(self, args, nargs) */ @@ -3314,9 +3301,8 @@ dummy_func( args--; total_args++; } - DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != - (METH_FASTCALL | METH_KEYWORDS), CALL); + DEOPT_IF(!PyCFunction_CheckExact(callable)); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS)); STAT_INC(CALL, hit); /* res = func(self, args, nargs, kwnames) */ _PyCFunctionFastWithKeywords cfunc = @@ -3341,9 +3327,9 @@ dummy_func( args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); + DEOPT_IF(total_args != 1); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.len, CALL); + DEOPT_IF(callable != interp->callable_cache.len); STAT_INC(CALL, hit); PyObject *arg = args[0]; Py_ssize_t len_i = PyObject_Length(arg); @@ -3365,9 +3351,9 @@ dummy_func( args--; total_args++; } - DEOPT_IF(total_args != 2, CALL); + DEOPT_IF(total_args != 2); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.isinstance, CALL); + DEOPT_IF(callable != interp->callable_cache.isinstance); STAT_INC(CALL, hit); PyObject *cls = args[1]; PyObject *inst = args[0]; @@ -3388,9 +3374,9 @@ dummy_func( inst(CALL_LIST_APPEND, (unused/1, unused/2, callable, self, args[oparg] -- unused)) { assert(oparg == 1); PyInterpreterState *interp = tstate->interp; - DEOPT_IF(callable != interp->callable_cache.list_append, CALL); + DEOPT_IF(callable != interp->callable_cache.list_append); assert(self != NULL); - DEOPT_IF(!PyList_Check(self), CALL); + DEOPT_IF(!PyList_Check(self)); STAT_INC(CALL, hit); if (_PyList_AppendTakeRef((PyListObject *)self, args[0]) < 0) { goto pop_1_error; // Since arg is DECREF'ed already @@ -3411,13 +3397,13 @@ dummy_func( total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(total_args != 2, CALL); - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + DEOPT_IF(total_args != 2); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_O, CALL); + DEOPT_IF(meth->ml_flags != METH_O); PyObject *arg = args[1]; PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; // This is slower but CPython promises to check all non-vectorcall @@ -3442,12 +3428,12 @@ dummy_func( total_args++; } PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS), CALL); + DEOPT_IF(meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)); PyTypeObject *d_type = method->d_common.d_type; PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, d_type)); STAT_INC(CALL, hit); int nargs = total_args - 1; _PyCFunctionFastWithKeywords cfunc = @@ -3471,13 +3457,13 @@ dummy_func( args--; total_args++; } - DEOPT_IF(total_args != 1, CALL); + DEOPT_IF(total_args != 1); PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); PyMethodDef *meth = method->d_method; PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); - DEOPT_IF(meth->ml_flags != METH_NOARGS, CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); + DEOPT_IF(meth->ml_flags != METH_NOARGS); STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; // This is slower but CPython promises to check all non-vectorcall @@ -3502,11 +3488,11 @@ dummy_func( } PyMethodDescrObject *method = (PyMethodDescrObject *)callable; /* Builtin METH_FASTCALL methods, without keywords */ - DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type), CALL); + DEOPT_IF(!Py_IS_TYPE(method, &PyMethodDescr_Type)); PyMethodDef *meth = method->d_method; - DEOPT_IF(meth->ml_flags != METH_FASTCALL, CALL); + DEOPT_IF(meth->ml_flags != METH_FASTCALL); PyObject *self = args[0]; - DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type), CALL); + DEOPT_IF(!Py_IS_TYPE(self, method->d_common.d_type)); STAT_INC(CALL, hit); _PyCFunctionFast cfunc = (_PyCFunctionFast)(void(*)(void))meth->ml_meth; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 55a03c9a23a572..662de57553fb47 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -13,8 +13,7 @@ _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif /* Possibly combine these two checks */ - DEOPT_IF(_PyFrame_GetCode(frame)->_co_instrumentation_version - != tstate->interp->monitoring_version, RESUME); + DEOPT_IF(_PyFrame_GetCode(frame)->_co_instrumentation_version != tstate->interp->monitoring_version, RESUME); DEOPT_IF(_Py_atomic_load_relaxed_int32(&tstate->interp->ceval.eval_breaker), RESUME); break; } @@ -236,8 +235,8 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left), _GUARD_BOTH_INT); + DEOPT_IF(!PyLong_CheckExact(right), _GUARD_BOTH_INT); break; } @@ -294,8 +293,8 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left), _GUARD_BOTH_FLOAT); + DEOPT_IF(!PyFloat_CheckExact(right), _GUARD_BOTH_FLOAT); break; } @@ -352,8 +351,8 @@ PyObject *left; right = stack_pointer[-1]; left = stack_pointer[-2]; - DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(left), _GUARD_BOTH_UNICODE); + DEOPT_IF(!PyUnicode_CheckExact(right), _GUARD_BOTH_UNICODE); break; } @@ -1189,8 +1188,8 @@ case _GUARD_GLOBALS_VERSION: { uint16_t version = (uint16_t)operand; PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), _GUARD_GLOBALS_VERSION); + DEOPT_IF(dict->ma_keys->dk_version != version, _GUARD_GLOBALS_VERSION); assert(DK_IS_UNICODE(dict->ma_keys)); break; } @@ -1198,8 +1197,8 @@ case _GUARD_BUILTINS_VERSION: { uint16_t version = (uint16_t)operand; PyDictObject *dict = (PyDictObject *)BUILTINS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), _GUARD_BUILTINS_VERSION); + DEOPT_IF(dict->ma_keys->dk_version != version, _GUARD_BUILTINS_VERSION); assert(DK_IS_UNICODE(dict->ma_keys)); break; } @@ -1211,7 +1210,7 @@ PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); res = entries[index].me_value; - DEOPT_IF(res == NULL, LOAD_GLOBAL); + DEOPT_IF(res == NULL, _LOAD_GLOBAL_MODULE); Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; @@ -1229,7 +1228,7 @@ PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); res = entries[index].me_value; - DEOPT_IF(res == NULL, LOAD_GLOBAL); + DEOPT_IF(res == NULL, _LOAD_GLOBAL_BUILTINS); Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; @@ -1679,7 +1678,7 @@ uint32_t type_version = (uint32_t)operand; PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version, _GUARD_TYPE_VERSION); break; } @@ -1689,9 +1688,7 @@ assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && - !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), - LOAD_ATTR); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), _CHECK_MANAGED_OBJECT_HAS_VALUES); break; } @@ -1703,7 +1700,7 @@ uint16_t index = (uint16_t)operand; PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); attr = _PyDictOrValues_GetValues(dorv)->values[index]; - DEOPT_IF(attr == NULL, LOAD_ATTR); + DEOPT_IF(attr == NULL, _LOAD_ATTR_INSTANCE_VALUE); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1722,7 +1719,7 @@ uint16_t index = (uint16_t)operand; char *addr = (char *)owner + index; attr = *(PyObject **)addr; - DEOPT_IF(attr == NULL, LOAD_ATTR); + DEOPT_IF(attr == NULL, _LOAD_ATTR_SLOT); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -1738,7 +1735,7 @@ owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); + DEOPT_IF(!_PyDictOrValues_IsValues(dorv), _GUARD_DORV_VALUES); break; } @@ -1770,7 +1767,7 @@ uint32_t type_version = (uint32_t)operand; PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version, _GUARD_TYPE_VERSION_STORE); break; } @@ -2114,7 +2111,7 @@ case _ITER_CHECK_LIST: { PyObject *iter; iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, _ITER_CHECK_LIST); break; } @@ -2159,7 +2156,7 @@ case _ITER_CHECK_TUPLE: { PyObject *iter; iter = stack_pointer[-1]; - DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, _ITER_CHECK_TUPLE); break; } @@ -2205,7 +2202,7 @@ PyObject *iter; iter = stack_pointer[-1]; _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, _ITER_CHECK_RANGE); break; } @@ -2300,9 +2297,7 @@ owner = stack_pointer[-1]; assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && - !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), - LOAD_ATTR); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT); break; } @@ -2312,8 +2307,7 @@ uint32_t keys_version = (uint32_t)operand; PyTypeObject *owner_cls = Py_TYPE(owner); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != - keys_version, LOAD_ATTR); + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, _GUARD_KEYS_VERSION); break; } @@ -2360,8 +2354,8 @@ PyObject *callable; null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); + DEOPT_IF(null != NULL, _CHECK_CALL_BOUND_METHOD_EXACT_ARGS); + DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, _CHECK_CALL_BOUND_METHOD_EXACT_ARGS); break; } @@ -2382,7 +2376,7 @@ } case _CHECK_PEP_523: { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, _CHECK_PEP_523); break; } @@ -2392,11 +2386,11 @@ self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; uint32_t func_version = (uint32_t)operand; - DEOPT_IF(!PyFunction_Check(callable), CALL); + DEOPT_IF(!PyFunction_Check(callable), _CHECK_FUNCTION_EXACT_ARGS); PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); + DEOPT_IF(func->func_version != func_version, _CHECK_FUNCTION_EXACT_ARGS); PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); + DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), _CHECK_FUNCTION_EXACT_ARGS); break; } @@ -2405,8 +2399,8 @@ callable = stack_pointer[-2 - oparg]; PyFunctionObject *func = (PyFunctionObject *)callable; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), _CHECK_STACK_SPACE); + DEOPT_IF(tstate->py_recursion_remaining <= 1, _CHECK_STACK_SPACE); break; } @@ -2671,8 +2665,7 @@ total_args++; } DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != - (METH_FASTCALL | METH_KEYWORDS), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL); STAT_INC(CALL, hit); /* res = func(self, args, nargs, kwnames) */ _PyCFunctionFastWithKeywords cfunc = diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 2701d416648a20..1a91c80cd12dcd 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -27,13 +27,13 @@ } TARGET(RESUME_CHECK) { + PREDICTED(RESUME_CHECK); #if defined(__EMSCRIPTEN__) DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif /* Possibly combine these two checks */ - DEOPT_IF(_PyFrame_GetCode(frame)->_co_instrumentation_version - != tstate->interp->monitoring_version, RESUME); + DEOPT_IF(_PyFrame_GetCode(frame)->_co_instrumentation_version != tstate->interp->monitoring_version, RESUME); DEOPT_IF(_Py_atomic_load_relaxed_int32(&tstate->interp->ceval.eval_breaker), RESUME); DISPATCH(); } @@ -283,6 +283,7 @@ } TARGET(TO_BOOL_BOOL) { + PREDICTED(TO_BOOL_BOOL); PyObject *value; value = stack_pointer[-1]; DEOPT_IF(!PyBool_Check(value), TO_BOOL); @@ -292,6 +293,7 @@ } TARGET(TO_BOOL_INT) { + PREDICTED(TO_BOOL_INT); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -311,6 +313,7 @@ } TARGET(TO_BOOL_LIST) { + PREDICTED(TO_BOOL_LIST); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -324,6 +327,7 @@ } TARGET(TO_BOOL_NONE) { + PREDICTED(TO_BOOL_NONE); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -337,6 +341,7 @@ } TARGET(TO_BOOL_STR) { + PREDICTED(TO_BOOL_STR); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -357,6 +362,7 @@ } TARGET(TO_BOOL_ALWAYS_TRUE) { + PREDICTED(TO_BOOL_ALWAYS_TRUE); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -384,6 +390,7 @@ } TARGET(BINARY_OP_MULTIPLY_INT) { + PREDICTED(BINARY_OP_MULTIPLY_INT); PyObject *right; PyObject *left; PyObject *res; @@ -391,8 +398,8 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); } // _BINARY_OP_MULTIPLY_INT { @@ -409,6 +416,7 @@ } TARGET(BINARY_OP_ADD_INT) { + PREDICTED(BINARY_OP_ADD_INT); PyObject *right; PyObject *left; PyObject *res; @@ -416,8 +424,8 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); } // _BINARY_OP_ADD_INT { @@ -434,6 +442,7 @@ } TARGET(BINARY_OP_SUBTRACT_INT) { + PREDICTED(BINARY_OP_SUBTRACT_INT); PyObject *right; PyObject *left; PyObject *res; @@ -441,8 +450,8 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP); } // _BINARY_OP_SUBTRACT_INT { @@ -459,6 +468,7 @@ } TARGET(BINARY_OP_MULTIPLY_FLOAT) { + PREDICTED(BINARY_OP_MULTIPLY_FLOAT); PyObject *right; PyObject *left; PyObject *res; @@ -466,8 +476,8 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); } // _BINARY_OP_MULTIPLY_FLOAT { @@ -484,6 +494,7 @@ } TARGET(BINARY_OP_ADD_FLOAT) { + PREDICTED(BINARY_OP_ADD_FLOAT); PyObject *right; PyObject *left; PyObject *res; @@ -491,8 +502,8 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); } // _BINARY_OP_ADD_FLOAT { @@ -509,6 +520,7 @@ } TARGET(BINARY_OP_SUBTRACT_FLOAT) { + PREDICTED(BINARY_OP_SUBTRACT_FLOAT); PyObject *right; PyObject *left; PyObject *res; @@ -516,8 +528,8 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP); } // _BINARY_OP_SUBTRACT_FLOAT { @@ -534,6 +546,7 @@ } TARGET(BINARY_OP_ADD_UNICODE) { + PREDICTED(BINARY_OP_ADD_UNICODE); PyObject *right; PyObject *left; PyObject *res; @@ -541,8 +554,8 @@ right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); } // _BINARY_OP_ADD_UNICODE { @@ -559,21 +572,22 @@ } TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { + PREDICTED(BINARY_OP_INPLACE_ADD_UNICODE); PyObject *right; PyObject *left; // _GUARD_BOTH_UNICODE right = stack_pointer[-1]; left = stack_pointer[-2]; { - DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); - DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP); + DEOPT_IF(!PyUnicode_CheckExact(right), BINARY_OP); } // _BINARY_OP_INPLACE_ADD_UNICODE { _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP]; assert(true_next.op.code == STORE_FAST); PyObject **target_local = &GETLOCAL(true_next.op.arg); - DEOPT_IF(*target_local != left, BINARY_OP); + DEOPT_IF(*target_local != left, BINARY_OP); STAT_INC(BINARY_OP, hit); /* Handle `left = left + right` or `left += right` for str. * @@ -677,6 +691,7 @@ } TARGET(BINARY_SUBSCR_LIST_INT) { + PREDICTED(BINARY_SUBSCR_LIST_INT); PyObject *sub; PyObject *list; PyObject *res; @@ -702,6 +717,7 @@ } TARGET(BINARY_SUBSCR_STR_INT) { + PREDICTED(BINARY_SUBSCR_STR_INT); PyObject *sub; PyObject *str; PyObject *res; @@ -726,6 +742,7 @@ } TARGET(BINARY_SUBSCR_TUPLE_INT) { + PREDICTED(BINARY_SUBSCR_TUPLE_INT); PyObject *sub; PyObject *tuple; PyObject *res; @@ -751,6 +768,7 @@ } TARGET(BINARY_SUBSCR_DICT) { + PREDICTED(BINARY_SUBSCR_DICT); PyObject *sub; PyObject *dict; PyObject *res; @@ -777,6 +795,7 @@ } TARGET(BINARY_SUBSCR_GETITEM) { + PREDICTED(BINARY_SUBSCR_GETITEM); PyObject *sub; PyObject *container; sub = stack_pointer[-1]; @@ -858,6 +877,7 @@ } TARGET(STORE_SUBSCR_LIST_INT) { + PREDICTED(STORE_SUBSCR_LIST_INT); PyObject *sub; PyObject *list; PyObject *value; @@ -886,6 +906,7 @@ } TARGET(STORE_SUBSCR_DICT) { + PREDICTED(STORE_SUBSCR_DICT); PyObject *sub; PyObject *dict; PyObject *value; @@ -1288,14 +1309,14 @@ } TARGET(SEND_GEN) { + PREDICTED(SEND_GEN); PyObject *v; PyObject *receiver; v = stack_pointer[-1]; receiver = stack_pointer[-2]; DEOPT_IF(tstate->interp->eval_frame, SEND); PyGenObject *gen = (PyGenObject *)receiver; - DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && - Py_TYPE(gen) != &PyCoro_Type, SEND); + DEOPT_IF(Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type, SEND); DEOPT_IF(gen->gi_frame_state >= FRAME_EXECUTING, SEND); STAT_INC(SEND, hit); _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; @@ -1524,6 +1545,7 @@ } TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { + PREDICTED(UNPACK_SEQUENCE_TWO_TUPLE); PyObject *seq; PyObject **values; seq = stack_pointer[-1]; @@ -1542,6 +1564,7 @@ } TARGET(UNPACK_SEQUENCE_TUPLE) { + PREDICTED(UNPACK_SEQUENCE_TUPLE); PyObject *seq; PyObject **values; seq = stack_pointer[-1]; @@ -1561,6 +1584,7 @@ } TARGET(UNPACK_SEQUENCE_LIST) { + PREDICTED(UNPACK_SEQUENCE_LIST); PyObject *seq; PyObject **values; seq = stack_pointer[-1]; @@ -1800,14 +1824,15 @@ } TARGET(LOAD_GLOBAL_MODULE) { + PREDICTED(LOAD_GLOBAL_MODULE); PyObject *res; PyObject *null = NULL; // _GUARD_GLOBALS_VERSION { uint16_t version = read_u16(&next_instr[1].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); assert(DK_IS_UNICODE(dict->ma_keys)); } // _LOAD_GLOBAL_MODULE @@ -1816,7 +1841,7 @@ PyDictObject *dict = (PyDictObject *)GLOBALS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(dict->ma_keys); res = entries[index].me_value; - DEOPT_IF(res == NULL, LOAD_GLOBAL); + DEOPT_IF(res == NULL, LOAD_GLOBAL); Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; @@ -1830,22 +1855,23 @@ } TARGET(LOAD_GLOBAL_BUILTIN) { + PREDICTED(LOAD_GLOBAL_BUILTIN); PyObject *res; PyObject *null = NULL; // _GUARD_GLOBALS_VERSION { uint16_t version = read_u16(&next_instr[1].cache); PyDictObject *dict = (PyDictObject *)GLOBALS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); assert(DK_IS_UNICODE(dict->ma_keys)); } // _GUARD_BUILTINS_VERSION { uint16_t version = read_u16(&next_instr[2].cache); PyDictObject *dict = (PyDictObject *)BUILTINS(); - DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); - DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); + DEOPT_IF(!PyDict_CheckExact(dict), LOAD_GLOBAL); + DEOPT_IF(dict->ma_keys->dk_version != version, LOAD_GLOBAL); assert(DK_IS_UNICODE(dict->ma_keys)); } // _LOAD_GLOBAL_BUILTINS @@ -1854,7 +1880,7 @@ PyDictObject *bdict = (PyDictObject *)BUILTINS(); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(bdict->ma_keys); res = entries[index].me_value; - DEOPT_IF(res == NULL, LOAD_GLOBAL); + DEOPT_IF(res == NULL, LOAD_GLOBAL); Py_INCREF(res); STAT_INC(LOAD_GLOBAL, hit); null = NULL; @@ -2276,6 +2302,7 @@ } TARGET(LOAD_SUPER_ATTR_ATTR) { + PREDICTED(LOAD_SUPER_ATTR_ATTR); PyObject *self; PyObject *class; PyObject *global_super; @@ -2300,6 +2327,7 @@ } TARGET(LOAD_SUPER_ATTR_METHOD) { + PREDICTED(LOAD_SUPER_ATTR_METHOD); PyObject *self; PyObject *class; PyObject *global_super; @@ -2394,6 +2422,7 @@ } TARGET(LOAD_ATTR_INSTANCE_VALUE) { + PREDICTED(LOAD_ATTR_INSTANCE_VALUE); PyObject *owner; PyObject *attr; PyObject *null = NULL; @@ -2403,23 +2432,21 @@ uint32_t type_version = read_u32(&next_instr[1].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } // _CHECK_MANAGED_OBJECT_HAS_VALUES { assert(Py_TYPE(owner)->tp_dictoffset < 0); assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && - !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), - LOAD_ATTR); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); } // _LOAD_ATTR_INSTANCE_VALUE { uint16_t index = read_u16(&next_instr[3].cache); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); attr = _PyDictOrValues_GetValues(dorv)->values[index]; - DEOPT_IF(attr == NULL, LOAD_ATTR); + DEOPT_IF(attr == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -2433,6 +2460,7 @@ } TARGET(LOAD_ATTR_MODULE) { + PREDICTED(LOAD_ATTR_MODULE); PyObject *owner; PyObject *attr; PyObject *null = NULL; @@ -2460,6 +2488,7 @@ } TARGET(LOAD_ATTR_WITH_HINT) { + PREDICTED(LOAD_ATTR_WITH_HINT); PyObject *owner; PyObject *attr; PyObject *null = NULL; @@ -2501,6 +2530,7 @@ } TARGET(LOAD_ATTR_SLOT) { + PREDICTED(LOAD_ATTR_SLOT); PyObject *owner; PyObject *attr; PyObject *null = NULL; @@ -2510,14 +2540,14 @@ uint32_t type_version = read_u32(&next_instr[1].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } // _LOAD_ATTR_SLOT { uint16_t index = read_u16(&next_instr[3].cache); char *addr = (char *)owner + index; attr = *(PyObject **)addr; - DEOPT_IF(attr == NULL, LOAD_ATTR); + DEOPT_IF(attr == NULL, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); Py_INCREF(attr); null = NULL; @@ -2531,6 +2561,7 @@ } TARGET(LOAD_ATTR_CLASS) { + PREDICTED(LOAD_ATTR_CLASS); PyObject *owner; PyObject *attr; PyObject *null = NULL; @@ -2539,8 +2570,7 @@ PyObject *descr = read_obj(&next_instr[5].cache); DEOPT_IF(!PyType_Check(owner), LOAD_ATTR); - DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, - LOAD_ATTR); + DEOPT_IF(((PyTypeObject *)owner)->tp_version_tag != type_version, LOAD_ATTR); assert(type_version != 0); STAT_INC(LOAD_ATTR, hit); @@ -2557,6 +2587,7 @@ } TARGET(LOAD_ATTR_PROPERTY) { + PREDICTED(LOAD_ATTR_PROPERTY); PyObject *owner; owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); @@ -2587,6 +2618,7 @@ } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { + PREDICTED(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); PyObject *owner; owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); @@ -2619,6 +2651,7 @@ } TARGET(STORE_ATTR_INSTANCE_VALUE) { + PREDICTED(STORE_ATTR_INSTANCE_VALUE); PyObject *owner; PyObject *value; // _GUARD_TYPE_VERSION_STORE @@ -2627,13 +2660,13 @@ uint32_t type_version = read_u32(&next_instr[1].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); } // _GUARD_DORV_VALUES { assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); + DEOPT_IF(!_PyDictOrValues_IsValues(dorv), STORE_ATTR); } // _STORE_ATTR_INSTANCE_VALUE value = stack_pointer[-2]; @@ -2658,6 +2691,7 @@ } TARGET(STORE_ATTR_WITH_HINT) { + PREDICTED(STORE_ATTR_WITH_HINT); PyObject *owner; PyObject *value; owner = stack_pointer[-1]; @@ -2708,6 +2742,7 @@ } TARGET(STORE_ATTR_SLOT) { + PREDICTED(STORE_ATTR_SLOT); PyObject *owner; PyObject *value; // _GUARD_TYPE_VERSION_STORE @@ -2716,7 +2751,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version, STORE_ATTR); } // _STORE_ATTR_SLOT value = stack_pointer[-2]; @@ -2770,6 +2805,7 @@ } TARGET(COMPARE_OP_FLOAT) { + PREDICTED(COMPARE_OP_FLOAT); PyObject *right; PyObject *left; PyObject *res; @@ -2793,6 +2829,7 @@ } TARGET(COMPARE_OP_INT) { + PREDICTED(COMPARE_OP_INT); PyObject *right; PyObject *left; PyObject *res; @@ -2820,6 +2857,7 @@ } TARGET(COMPARE_OP_STR) { + PREDICTED(COMPARE_OP_STR); PyObject *right; PyObject *left; PyObject *res; @@ -3292,12 +3330,13 @@ } TARGET(FOR_ITER_LIST) { + PREDICTED(FOR_ITER_LIST); PyObject *iter; PyObject *next; // _ITER_CHECK_LIST iter = stack_pointer[-1]; { - DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(iter) != &PyListIter_Type, FOR_ITER); } // _ITER_JUMP_LIST { @@ -3334,12 +3373,13 @@ } TARGET(FOR_ITER_TUPLE) { + PREDICTED(FOR_ITER_TUPLE); PyObject *iter; PyObject *next; // _ITER_CHECK_TUPLE iter = stack_pointer[-1]; { - DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(iter) != &PyTupleIter_Type, FOR_ITER); } // _ITER_JUMP_TUPLE { @@ -3376,13 +3416,14 @@ } TARGET(FOR_ITER_RANGE) { + PREDICTED(FOR_ITER_RANGE); PyObject *iter; PyObject *next; // _ITER_CHECK_RANGE iter = stack_pointer[-1]; { _PyRangeIterObject *r = (_PyRangeIterObject *)iter; - DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); + DEOPT_IF(Py_TYPE(r) != &PyRangeIter_Type, FOR_ITER); } // _ITER_JUMP_RANGE { @@ -3416,6 +3457,7 @@ } TARGET(FOR_ITER_GEN) { + PREDICTED(FOR_ITER_GEN); PyObject *iter; iter = stack_pointer[-1]; DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); @@ -3576,6 +3618,7 @@ } TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { + PREDICTED(LOAD_ATTR_METHOD_WITH_VALUES); PyObject *owner; PyObject *attr; PyObject *self; @@ -3585,23 +3628,20 @@ uint32_t type_version = read_u32(&next_instr[1].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT { assert(Py_TYPE(owner)->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && - !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), - LOAD_ATTR); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); } // _GUARD_KEYS_VERSION { uint32_t keys_version = read_u32(&next_instr[3].cache); PyTypeObject *owner_cls = Py_TYPE(owner); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != - keys_version, LOAD_ATTR); + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); } // _LOAD_ATTR_METHOD_WITH_VALUES { @@ -3622,6 +3662,7 @@ } TARGET(LOAD_ATTR_METHOD_NO_DICT) { + PREDICTED(LOAD_ATTR_METHOD_NO_DICT); PyObject *owner; PyObject *attr; PyObject *self; @@ -3631,7 +3672,7 @@ uint32_t type_version = read_u32(&next_instr[1].cache); PyTypeObject *tp = Py_TYPE(owner); assert(type_version != 0); - DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); + DEOPT_IF(tp->tp_version_tag != type_version, LOAD_ATTR); } // _LOAD_ATTR_METHOD_NO_DICT { @@ -3652,6 +3693,7 @@ } TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { + PREDICTED(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); PyObject *owner; PyObject *attr; owner = stack_pointer[-1]; @@ -3664,12 +3706,9 @@ DEOPT_IF(owner_cls->tp_version_tag != type_version, LOAD_ATTR); assert(owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT); PyDictOrValues *dorv = _PyObject_DictOrValuesPointer(owner); - DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && - !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), - LOAD_ATTR); + DEOPT_IF(!_PyDictOrValues_IsValues(*dorv) && !_PyObject_MakeInstanceAttributesFromDict(owner, dorv), LOAD_ATTR); PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != - keys_version, LOAD_ATTR); + DEOPT_IF(owner_heap_type->ht_cached_keys->dk_version != keys_version, LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); Py_DECREF(owner); @@ -3680,6 +3719,7 @@ } TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { + PREDICTED(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); PyObject *owner; PyObject *attr; owner = stack_pointer[-1]; @@ -3700,6 +3740,7 @@ } TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { + PREDICTED(LOAD_ATTR_METHOD_LAZY_DICT); PyObject *owner; PyObject *attr; PyObject *self; @@ -3836,6 +3877,7 @@ } TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { + PREDICTED(CALL_BOUND_METHOD_EXACT_ARGS); PyObject *null; PyObject *callable; PyObject *self; @@ -3845,14 +3887,14 @@ _PyInterpreterFrame *new_frame; // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL); } // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; { - DEOPT_IF(null != NULL, CALL); - DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); + DEOPT_IF(null != NULL, CALL); + DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); } // _INIT_CALL_BOUND_METHOD_EXACT_ARGS { @@ -3868,18 +3910,18 @@ callable = func; { uint32_t func_version = read_u32(&next_instr[1].cache); - DEOPT_IF(!PyFunction_Check(callable), CALL); + DEOPT_IF(!PyFunction_Check(callable), CALL); PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); + DEOPT_IF(func->func_version != func_version, CALL); PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); + DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); } // _CHECK_STACK_SPACE { PyFunctionObject *func = (PyFunctionObject *)callable; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); } // _INIT_CALL_PY_EXACT_ARGS args = stack_pointer - oparg; @@ -3933,31 +3975,32 @@ } TARGET(CALL_PY_EXACT_ARGS) { + PREDICTED(CALL_PY_EXACT_ARGS); PyObject *self_or_null; PyObject *callable; PyObject **args; _PyInterpreterFrame *new_frame; // _CHECK_PEP_523 { - DEOPT_IF(tstate->interp->eval_frame, CALL); + DEOPT_IF(tstate->interp->eval_frame, CALL); } // _CHECK_FUNCTION_EXACT_ARGS self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; { uint32_t func_version = read_u32(&next_instr[1].cache); - DEOPT_IF(!PyFunction_Check(callable), CALL); + DEOPT_IF(!PyFunction_Check(callable), CALL); PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); + DEOPT_IF(func->func_version != func_version, CALL); PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); + DEOPT_IF(code->co_argcount != oparg + (self_or_null != NULL), CALL); } // _CHECK_STACK_SPACE { PyFunctionObject *func = (PyFunctionObject *)callable; PyCodeObject *code = (PyCodeObject *)func->func_code; - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); + DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); + DEOPT_IF(tstate->py_recursion_remaining <= 1, CALL); } // _INIT_CALL_PY_EXACT_ARGS args = stack_pointer - oparg; @@ -4011,6 +4054,7 @@ } TARGET(CALL_PY_WITH_DEFAULTS) { + PREDICTED(CALL_PY_WITH_DEFAULTS); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4053,6 +4097,7 @@ } TARGET(CALL_TYPE_1) { + PREDICTED(CALL_TYPE_1); PyObject **args; PyObject *null; PyObject *callable; @@ -4076,6 +4121,7 @@ } TARGET(CALL_STR_1) { + PREDICTED(CALL_STR_1); PyObject **args; PyObject *null; PyObject *callable; @@ -4101,6 +4147,7 @@ } TARGET(CALL_TUPLE_1) { + PREDICTED(CALL_TUPLE_1); PyObject **args; PyObject *null; PyObject *callable; @@ -4126,6 +4173,7 @@ } TARGET(CALL_ALLOC_AND_ENTER_INIT) { + PREDICTED(CALL_ALLOC_AND_ENTER_INIT); PyObject **args; PyObject *null; PyObject *callable; @@ -4198,6 +4246,7 @@ } TARGET(CALL_BUILTIN_CLASS) { + PREDICTED(CALL_BUILTIN_CLASS); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4230,6 +4279,7 @@ } TARGET(CALL_BUILTIN_O) { + PREDICTED(CALL_BUILTIN_O); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4270,6 +4320,7 @@ } TARGET(CALL_BUILTIN_FAST) { + PREDICTED(CALL_BUILTIN_FAST); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4314,6 +4365,7 @@ } TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { + PREDICTED(CALL_BUILTIN_FAST_WITH_KEYWORDS); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4328,8 +4380,7 @@ total_args++; } DEOPT_IF(!PyCFunction_CheckExact(callable), CALL); - DEOPT_IF(PyCFunction_GET_FLAGS(callable) != - (METH_FASTCALL | METH_KEYWORDS), CALL); + DEOPT_IF(PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS), CALL); STAT_INC(CALL, hit); /* res = func(self, args, nargs, kwnames) */ _PyCFunctionFastWithKeywords cfunc = @@ -4353,6 +4404,7 @@ } TARGET(CALL_LEN) { + PREDICTED(CALL_LEN); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4389,6 +4441,7 @@ } TARGET(CALL_ISINSTANCE) { + PREDICTED(CALL_ISINSTANCE); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4427,6 +4480,7 @@ } TARGET(CALL_LIST_APPEND) { + PREDICTED(CALL_LIST_APPEND); PyObject **args; PyObject *self; PyObject *callable; @@ -4452,6 +4506,7 @@ } TARGET(CALL_METHOD_DESCRIPTOR_O) { + PREDICTED(CALL_METHOD_DESCRIPTOR_O); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4495,6 +4550,7 @@ } TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { + PREDICTED(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4536,6 +4592,7 @@ } TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { + PREDICTED(CALL_METHOD_DESCRIPTOR_NOARGS); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4578,6 +4635,7 @@ } TARGET(CALL_METHOD_DESCRIPTOR_FAST) { + PREDICTED(CALL_METHOD_DESCRIPTOR_FAST); PyObject **args; PyObject *self_or_null; PyObject *callable; diff --git a/Tools/cases_generator/analysis.py b/Tools/cases_generator/analysis.py index 91dcba8ceee13d..181ff5d89b6c0d 100644 --- a/Tools/cases_generator/analysis.py +++ b/Tools/cases_generator/analysis.py @@ -26,7 +26,7 @@ "co_names": "Use FRAME_CO_NAMES.", } -RE_PREDICTED = r"^\s*(?:GO_TO_INSTRUCTION\(|DEOPT_IF\(.*?,\s*)(\w+)\);\s*(?://.*)?$" +RE_PREDICTED = r"^\s*GO_TO_INSTRUCTION\((\w+)\);\s*(?://.*)?$" class Analyzer: diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 9192d1038ab7d6..0b998d109e9548 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -781,9 +781,17 @@ def write_instructions( case parsing.Macro(): n_macros += 1 mac = self.macro_instrs[thing.name] - stacking.write_macro_instr( - mac, self.out, self.families.get(mac.name) - ) + family = self.families.get(mac.name) + if family is None: + for family in self.families.values(): + if mac.name in family.members: + break + else: + family = None + # A sanctioned exception + if mac.name == "BINARY_OP_INPLACE_ADD_UNICODE": + family = self.families["BINARY_OP"] + stacking.write_macro_instr(mac, self.out, family) case parsing.Pseudo(): pass case _: diff --git a/Tools/cases_generator/instructions.py b/Tools/cases_generator/instructions.py index 6fbf7d93f42fde..1452ab07390aec 100644 --- a/Tools/cases_generator/instructions.py +++ b/Tools/cases_generator/instructions.py @@ -144,7 +144,8 @@ def write_body( out: Formatter, dedent: int, active_caches: list[ActiveCacheEffect], - tier: Tiers = TIER_ONE, + tier: Tiers, + family: parsing.Family | None, ) -> None: """Write the instruction body.""" # Write cache effect variable declarations and initializations @@ -207,6 +208,14 @@ def write_body( ) else: out.write_raw(f"{space}if ({cond}) goto {label};\n") + elif m := re.match(r"(\s*)DEOPT_IF\((.+)\);\s*(?://.*)?$", line): + space, cond = m.groups() + target = family.name if family else self.name + out.write_raw(f"{space}DEOPT_IF({cond}, {target});\n") + elif "DEOPT" in line: + filename = context.owner.filename + lineno = context.owner.tokens[context.begin].line + print(f"{filename}:{lineno}: ERROR: DEOPT_IF() must be all on one line") elif m := re.match(r"(\s*)DECREF_INPUTS\(\);\s*(?://.*)?$", line): out.reset_lineno() space = extra + m.group(1) @@ -244,7 +253,8 @@ def write_body( out: Formatter, dedent: int, active_caches: list[ActiveCacheEffect], - tier: Tiers = TIER_ONE, + tier: Tiers, + family: parsing.Family | None, ) -> None: pass diff --git a/Tools/cases_generator/stacking.py b/Tools/cases_generator/stacking.py index 1f9fda66a5f034..4e807337411ac3 100644 --- a/Tools/cases_generator/stacking.py +++ b/Tools/cases_generator/stacking.py @@ -351,6 +351,7 @@ def write_single_instr( out, tier, 0, + instr.family, ) except AssertionError as err: raise AssertionError(f"Error writing instruction {instr.name}") from err @@ -366,11 +367,13 @@ def write_macro_instr( ] out.emit("") with out.block(f"TARGET({mac.name})"): - if mac.predicted: + if mac.predicted or family is not None: out.emit(f"PREDICTED({mac.name});") out.static_assert_family_size(mac.name, family, mac.cache_offset) try: - next_instr_is_set = write_components(parts, out, TIER_ONE, mac.cache_offset) + next_instr_is_set = write_components( + parts, out, TIER_ONE, mac.cache_offset, family + ) except AssertionError as err: raise AssertionError(f"Error writing macro {mac.name}") from err if not parts[-1].instr.always_exits: @@ -386,6 +389,7 @@ def write_components( out: Formatter, tier: Tiers, cache_offset: int, + family: Family | None, ) -> bool: managers = get_managers(parts) @@ -454,10 +458,10 @@ def write_components( assert_no_pokes(managers) if len(parts) == 1: - mgr.instr.write_body(out, 0, mgr.active_caches, tier) + mgr.instr.write_body(out, 0, mgr.active_caches, tier, family) else: with out.block(""): - mgr.instr.write_body(out, -4, mgr.active_caches, tier) + mgr.instr.write_body(out, -4, mgr.active_caches, tier, family) if mgr is managers[-1] and not next_instr_is_set and not mgr.instr.always_exits: # Adjust the stack to its final depth, *then* write the From 3428c84056d2b16f064a9684f4e7c95237c96d4f Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sun, 1 Oct 2023 16:50:11 -0700 Subject: [PATCH 2/4] Fix tests and fix bug they found --- Lib/test/test_generated_cases.py | 11 ++++- Python/generated_cases.c.h | 70 ------------------------------- Tools/cases_generator/analysis.py | 4 +- Tools/cases_generator/stacking.py | 2 +- 4 files changed, 12 insertions(+), 75 deletions(-) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index b5eaf824aee706..5971d2e436e4aa 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -239,16 +239,22 @@ def test_overlap(self): def test_predictions_and_eval_breaker(self): input = """ - inst(OP1, (--)) { + inst(OP1, (arg -- rest)) { } inst(OP3, (arg -- res)) { - DEOPT_IF(xxx, OP1); + DEOPT_IF(xxx); CHECK_EVAL_BREAKER(); } + family(OP1, INLINE_CACHE_ENTRIES_OP1) = { OP3 }; """ output = """ TARGET(OP1) { PREDICTED(OP1); + static_assert(INLINE_CACHE_ENTRIES_OP1 == 0, "incorrect cache size"); + PyObject *arg; + PyObject *rest; + arg = stack_pointer[-1]; + stack_pointer[-1] = rest; DISPATCH(); } @@ -371,6 +377,7 @@ def test_macro_instruction(self): } TARGET(OP) { + PREDICTED(OP); static_assert(INLINE_CACHE_ENTRIES_OP == 5, "incorrect cache size"); PyObject *right; PyObject *left; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 1a91c80cd12dcd..96d9d9db9c7928 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -27,7 +27,6 @@ } TARGET(RESUME_CHECK) { - PREDICTED(RESUME_CHECK); #if defined(__EMSCRIPTEN__) DEOPT_IF(_Py_emscripten_signal_clock == 0, RESUME); _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; @@ -283,7 +282,6 @@ } TARGET(TO_BOOL_BOOL) { - PREDICTED(TO_BOOL_BOOL); PyObject *value; value = stack_pointer[-1]; DEOPT_IF(!PyBool_Check(value), TO_BOOL); @@ -293,7 +291,6 @@ } TARGET(TO_BOOL_INT) { - PREDICTED(TO_BOOL_INT); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -313,7 +310,6 @@ } TARGET(TO_BOOL_LIST) { - PREDICTED(TO_BOOL_LIST); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -327,7 +323,6 @@ } TARGET(TO_BOOL_NONE) { - PREDICTED(TO_BOOL_NONE); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -341,7 +336,6 @@ } TARGET(TO_BOOL_STR) { - PREDICTED(TO_BOOL_STR); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -362,7 +356,6 @@ } TARGET(TO_BOOL_ALWAYS_TRUE) { - PREDICTED(TO_BOOL_ALWAYS_TRUE); PyObject *value; PyObject *res; value = stack_pointer[-1]; @@ -390,7 +383,6 @@ } TARGET(BINARY_OP_MULTIPLY_INT) { - PREDICTED(BINARY_OP_MULTIPLY_INT); PyObject *right; PyObject *left; PyObject *res; @@ -416,7 +408,6 @@ } TARGET(BINARY_OP_ADD_INT) { - PREDICTED(BINARY_OP_ADD_INT); PyObject *right; PyObject *left; PyObject *res; @@ -442,7 +433,6 @@ } TARGET(BINARY_OP_SUBTRACT_INT) { - PREDICTED(BINARY_OP_SUBTRACT_INT); PyObject *right; PyObject *left; PyObject *res; @@ -468,7 +458,6 @@ } TARGET(BINARY_OP_MULTIPLY_FLOAT) { - PREDICTED(BINARY_OP_MULTIPLY_FLOAT); PyObject *right; PyObject *left; PyObject *res; @@ -494,7 +483,6 @@ } TARGET(BINARY_OP_ADD_FLOAT) { - PREDICTED(BINARY_OP_ADD_FLOAT); PyObject *right; PyObject *left; PyObject *res; @@ -520,7 +508,6 @@ } TARGET(BINARY_OP_SUBTRACT_FLOAT) { - PREDICTED(BINARY_OP_SUBTRACT_FLOAT); PyObject *right; PyObject *left; PyObject *res; @@ -546,7 +533,6 @@ } TARGET(BINARY_OP_ADD_UNICODE) { - PREDICTED(BINARY_OP_ADD_UNICODE); PyObject *right; PyObject *left; PyObject *res; @@ -572,7 +558,6 @@ } TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { - PREDICTED(BINARY_OP_INPLACE_ADD_UNICODE); PyObject *right; PyObject *left; // _GUARD_BOTH_UNICODE @@ -691,7 +676,6 @@ } TARGET(BINARY_SUBSCR_LIST_INT) { - PREDICTED(BINARY_SUBSCR_LIST_INT); PyObject *sub; PyObject *list; PyObject *res; @@ -717,7 +701,6 @@ } TARGET(BINARY_SUBSCR_STR_INT) { - PREDICTED(BINARY_SUBSCR_STR_INT); PyObject *sub; PyObject *str; PyObject *res; @@ -742,7 +725,6 @@ } TARGET(BINARY_SUBSCR_TUPLE_INT) { - PREDICTED(BINARY_SUBSCR_TUPLE_INT); PyObject *sub; PyObject *tuple; PyObject *res; @@ -768,7 +750,6 @@ } TARGET(BINARY_SUBSCR_DICT) { - PREDICTED(BINARY_SUBSCR_DICT); PyObject *sub; PyObject *dict; PyObject *res; @@ -795,7 +776,6 @@ } TARGET(BINARY_SUBSCR_GETITEM) { - PREDICTED(BINARY_SUBSCR_GETITEM); PyObject *sub; PyObject *container; sub = stack_pointer[-1]; @@ -877,7 +857,6 @@ } TARGET(STORE_SUBSCR_LIST_INT) { - PREDICTED(STORE_SUBSCR_LIST_INT); PyObject *sub; PyObject *list; PyObject *value; @@ -906,7 +885,6 @@ } TARGET(STORE_SUBSCR_DICT) { - PREDICTED(STORE_SUBSCR_DICT); PyObject *sub; PyObject *dict; PyObject *value; @@ -1309,7 +1287,6 @@ } TARGET(SEND_GEN) { - PREDICTED(SEND_GEN); PyObject *v; PyObject *receiver; v = stack_pointer[-1]; @@ -1545,7 +1522,6 @@ } TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { - PREDICTED(UNPACK_SEQUENCE_TWO_TUPLE); PyObject *seq; PyObject **values; seq = stack_pointer[-1]; @@ -1564,7 +1540,6 @@ } TARGET(UNPACK_SEQUENCE_TUPLE) { - PREDICTED(UNPACK_SEQUENCE_TUPLE); PyObject *seq; PyObject **values; seq = stack_pointer[-1]; @@ -1584,7 +1559,6 @@ } TARGET(UNPACK_SEQUENCE_LIST) { - PREDICTED(UNPACK_SEQUENCE_LIST); PyObject *seq; PyObject **values; seq = stack_pointer[-1]; @@ -1824,7 +1798,6 @@ } TARGET(LOAD_GLOBAL_MODULE) { - PREDICTED(LOAD_GLOBAL_MODULE); PyObject *res; PyObject *null = NULL; // _GUARD_GLOBALS_VERSION @@ -1855,7 +1828,6 @@ } TARGET(LOAD_GLOBAL_BUILTIN) { - PREDICTED(LOAD_GLOBAL_BUILTIN); PyObject *res; PyObject *null = NULL; // _GUARD_GLOBALS_VERSION @@ -2302,7 +2274,6 @@ } TARGET(LOAD_SUPER_ATTR_ATTR) { - PREDICTED(LOAD_SUPER_ATTR_ATTR); PyObject *self; PyObject *class; PyObject *global_super; @@ -2327,7 +2298,6 @@ } TARGET(LOAD_SUPER_ATTR_METHOD) { - PREDICTED(LOAD_SUPER_ATTR_METHOD); PyObject *self; PyObject *class; PyObject *global_super; @@ -2422,7 +2392,6 @@ } TARGET(LOAD_ATTR_INSTANCE_VALUE) { - PREDICTED(LOAD_ATTR_INSTANCE_VALUE); PyObject *owner; PyObject *attr; PyObject *null = NULL; @@ -2460,7 +2429,6 @@ } TARGET(LOAD_ATTR_MODULE) { - PREDICTED(LOAD_ATTR_MODULE); PyObject *owner; PyObject *attr; PyObject *null = NULL; @@ -2488,7 +2456,6 @@ } TARGET(LOAD_ATTR_WITH_HINT) { - PREDICTED(LOAD_ATTR_WITH_HINT); PyObject *owner; PyObject *attr; PyObject *null = NULL; @@ -2530,7 +2497,6 @@ } TARGET(LOAD_ATTR_SLOT) { - PREDICTED(LOAD_ATTR_SLOT); PyObject *owner; PyObject *attr; PyObject *null = NULL; @@ -2561,7 +2527,6 @@ } TARGET(LOAD_ATTR_CLASS) { - PREDICTED(LOAD_ATTR_CLASS); PyObject *owner; PyObject *attr; PyObject *null = NULL; @@ -2587,7 +2552,6 @@ } TARGET(LOAD_ATTR_PROPERTY) { - PREDICTED(LOAD_ATTR_PROPERTY); PyObject *owner; owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); @@ -2618,7 +2582,6 @@ } TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { - PREDICTED(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); PyObject *owner; owner = stack_pointer[-1]; uint32_t type_version = read_u32(&next_instr[1].cache); @@ -2651,7 +2614,6 @@ } TARGET(STORE_ATTR_INSTANCE_VALUE) { - PREDICTED(STORE_ATTR_INSTANCE_VALUE); PyObject *owner; PyObject *value; // _GUARD_TYPE_VERSION_STORE @@ -2691,7 +2653,6 @@ } TARGET(STORE_ATTR_WITH_HINT) { - PREDICTED(STORE_ATTR_WITH_HINT); PyObject *owner; PyObject *value; owner = stack_pointer[-1]; @@ -2742,7 +2703,6 @@ } TARGET(STORE_ATTR_SLOT) { - PREDICTED(STORE_ATTR_SLOT); PyObject *owner; PyObject *value; // _GUARD_TYPE_VERSION_STORE @@ -2805,7 +2765,6 @@ } TARGET(COMPARE_OP_FLOAT) { - PREDICTED(COMPARE_OP_FLOAT); PyObject *right; PyObject *left; PyObject *res; @@ -2829,7 +2788,6 @@ } TARGET(COMPARE_OP_INT) { - PREDICTED(COMPARE_OP_INT); PyObject *right; PyObject *left; PyObject *res; @@ -2857,7 +2815,6 @@ } TARGET(COMPARE_OP_STR) { - PREDICTED(COMPARE_OP_STR); PyObject *right; PyObject *left; PyObject *res; @@ -3330,7 +3287,6 @@ } TARGET(FOR_ITER_LIST) { - PREDICTED(FOR_ITER_LIST); PyObject *iter; PyObject *next; // _ITER_CHECK_LIST @@ -3373,7 +3329,6 @@ } TARGET(FOR_ITER_TUPLE) { - PREDICTED(FOR_ITER_TUPLE); PyObject *iter; PyObject *next; // _ITER_CHECK_TUPLE @@ -3416,7 +3371,6 @@ } TARGET(FOR_ITER_RANGE) { - PREDICTED(FOR_ITER_RANGE); PyObject *iter; PyObject *next; // _ITER_CHECK_RANGE @@ -3457,7 +3411,6 @@ } TARGET(FOR_ITER_GEN) { - PREDICTED(FOR_ITER_GEN); PyObject *iter; iter = stack_pointer[-1]; DEOPT_IF(tstate->interp->eval_frame, FOR_ITER); @@ -3618,7 +3571,6 @@ } TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { - PREDICTED(LOAD_ATTR_METHOD_WITH_VALUES); PyObject *owner; PyObject *attr; PyObject *self; @@ -3662,7 +3614,6 @@ } TARGET(LOAD_ATTR_METHOD_NO_DICT) { - PREDICTED(LOAD_ATTR_METHOD_NO_DICT); PyObject *owner; PyObject *attr; PyObject *self; @@ -3693,7 +3644,6 @@ } TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { - PREDICTED(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); PyObject *owner; PyObject *attr; owner = stack_pointer[-1]; @@ -3719,7 +3669,6 @@ } TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { - PREDICTED(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); PyObject *owner; PyObject *attr; owner = stack_pointer[-1]; @@ -3740,7 +3689,6 @@ } TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { - PREDICTED(LOAD_ATTR_METHOD_LAZY_DICT); PyObject *owner; PyObject *attr; PyObject *self; @@ -3877,7 +3825,6 @@ } TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { - PREDICTED(CALL_BOUND_METHOD_EXACT_ARGS); PyObject *null; PyObject *callable; PyObject *self; @@ -3975,7 +3922,6 @@ } TARGET(CALL_PY_EXACT_ARGS) { - PREDICTED(CALL_PY_EXACT_ARGS); PyObject *self_or_null; PyObject *callable; PyObject **args; @@ -4054,7 +4000,6 @@ } TARGET(CALL_PY_WITH_DEFAULTS) { - PREDICTED(CALL_PY_WITH_DEFAULTS); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4097,7 +4042,6 @@ } TARGET(CALL_TYPE_1) { - PREDICTED(CALL_TYPE_1); PyObject **args; PyObject *null; PyObject *callable; @@ -4121,7 +4065,6 @@ } TARGET(CALL_STR_1) { - PREDICTED(CALL_STR_1); PyObject **args; PyObject *null; PyObject *callable; @@ -4147,7 +4090,6 @@ } TARGET(CALL_TUPLE_1) { - PREDICTED(CALL_TUPLE_1); PyObject **args; PyObject *null; PyObject *callable; @@ -4173,7 +4115,6 @@ } TARGET(CALL_ALLOC_AND_ENTER_INIT) { - PREDICTED(CALL_ALLOC_AND_ENTER_INIT); PyObject **args; PyObject *null; PyObject *callable; @@ -4246,7 +4187,6 @@ } TARGET(CALL_BUILTIN_CLASS) { - PREDICTED(CALL_BUILTIN_CLASS); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4279,7 +4219,6 @@ } TARGET(CALL_BUILTIN_O) { - PREDICTED(CALL_BUILTIN_O); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4320,7 +4259,6 @@ } TARGET(CALL_BUILTIN_FAST) { - PREDICTED(CALL_BUILTIN_FAST); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4365,7 +4303,6 @@ } TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { - PREDICTED(CALL_BUILTIN_FAST_WITH_KEYWORDS); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4404,7 +4341,6 @@ } TARGET(CALL_LEN) { - PREDICTED(CALL_LEN); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4441,7 +4377,6 @@ } TARGET(CALL_ISINSTANCE) { - PREDICTED(CALL_ISINSTANCE); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4480,7 +4415,6 @@ } TARGET(CALL_LIST_APPEND) { - PREDICTED(CALL_LIST_APPEND); PyObject **args; PyObject *self; PyObject *callable; @@ -4506,7 +4440,6 @@ } TARGET(CALL_METHOD_DESCRIPTOR_O) { - PREDICTED(CALL_METHOD_DESCRIPTOR_O); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4550,7 +4483,6 @@ } TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { - PREDICTED(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4592,7 +4524,6 @@ } TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { - PREDICTED(CALL_METHOD_DESCRIPTOR_NOARGS); PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -4635,7 +4566,6 @@ } TARGET(CALL_METHOD_DESCRIPTOR_FAST) { - PREDICTED(CALL_METHOD_DESCRIPTOR_FAST); PyObject **args; PyObject *self_or_null; PyObject *callable; diff --git a/Tools/cases_generator/analysis.py b/Tools/cases_generator/analysis.py index 181ff5d89b6c0d..11d3903cd1e050 100644 --- a/Tools/cases_generator/analysis.py +++ b/Tools/cases_generator/analysis.py @@ -26,7 +26,7 @@ "co_names": "Use FRAME_CO_NAMES.", } -RE_PREDICTED = r"^\s*GO_TO_INSTRUCTION\((\w+)\);\s*(?://.*)?$" +RE_GO_TO_INSTR = r"^\s*GO_TO_INSTRUCTION\((\w+)\);\s*(?://.*)?$" class Analyzer: @@ -196,7 +196,7 @@ def find_predictions(self) -> None: for instr in self.instrs.values(): targets: set[str] = set() for line in instr.block_text: - if m := re.match(RE_PREDICTED, line): + if m := re.match(RE_GO_TO_INSTR, line): targets.add(m.group(1)) for target in targets: if target_instr := self.instrs.get(target): diff --git a/Tools/cases_generator/stacking.py b/Tools/cases_generator/stacking.py index 4e807337411ac3..ef0571aabd7a97 100644 --- a/Tools/cases_generator/stacking.py +++ b/Tools/cases_generator/stacking.py @@ -367,7 +367,7 @@ def write_macro_instr( ] out.emit("") with out.block(f"TARGET({mac.name})"): - if mac.predicted or family is not None: + if mac.predicted or (family is not None and mac.name == family.name): out.emit(f"PREDICTED({mac.name});") out.static_assert_family_size(mac.name, family, mac.cache_offset) try: From 785c3739f91679bccea7aaae4420519f2edb521f Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 2 Oct 2023 11:03:59 -0700 Subject: [PATCH 3/4] Always set/use .predicted flag --- Tools/cases_generator/analysis.py | 13 ++++++++++--- Tools/cases_generator/stacking.py | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/Tools/cases_generator/analysis.py b/Tools/cases_generator/analysis.py index 11d3903cd1e050..68d141b7d1abfc 100644 --- a/Tools/cases_generator/analysis.py +++ b/Tools/cases_generator/analysis.py @@ -187,12 +187,19 @@ def analyze(self) -> None: Raises SystemExit if there is an error. """ self.analyze_macros_and_pseudos() - self.find_predictions() self.map_families() + self.mark_predictions() self.check_families() - def find_predictions(self) -> None: - """Find the instructions that need PREDICTED() labels.""" + def mark_predictions(self) -> None: + """Mark the instructions that need PREDICTED() labels.""" + # Start with family heads + for family in self.families.values(): + if family.name in self.instrs: + self.instrs[family.name].predicted = True + if family.name in self.macro_instrs: + self.macro_instrs[family.name].predicted = True + # Also look for GO_TO_INSTRUCTION() calls for instr in self.instrs.values(): targets: set[str] = set() for line in instr.block_text: diff --git a/Tools/cases_generator/stacking.py b/Tools/cases_generator/stacking.py index ef0571aabd7a97..32a8d94887ef7e 100644 --- a/Tools/cases_generator/stacking.py +++ b/Tools/cases_generator/stacking.py @@ -367,7 +367,7 @@ def write_macro_instr( ] out.emit("") with out.block(f"TARGET({mac.name})"): - if mac.predicted or (family is not None and mac.name == family.name): + if mac.predicted: out.emit(f"PREDICTED({mac.name});") out.static_assert_family_size(mac.name, family, mac.cache_offset) try: From 097909ee63af7423adf9f87001c49c05582194c7 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 2 Oct 2023 12:37:39 -0700 Subject: [PATCH 4/4] Add MacroInstruction.family field and use it --- Tools/cases_generator/analysis.py | 9 ++++++++- Tools/cases_generator/generate_cases.py | 12 +----------- Tools/cases_generator/instructions.py | 2 ++ Tools/cases_generator/stacking.py | 8 +++----- 4 files changed, 14 insertions(+), 17 deletions(-) diff --git a/Tools/cases_generator/analysis.py b/Tools/cases_generator/analysis.py index 68d141b7d1abfc..7bbc924e5083f1 100644 --- a/Tools/cases_generator/analysis.py +++ b/Tools/cases_generator/analysis.py @@ -232,11 +232,18 @@ def map_families(self) -> None: ) else: member_instr.family = family - elif not self.macro_instrs.get(member): + if member_mac := self.macro_instrs.get(member): + assert member_mac.family is None, (member, member_mac.family.name) + member_mac.family = family + if not member_instr and not member_mac: self.error( f"Unknown instruction {member!r} referenced in family {family.name!r}", family, ) + # A sanctioned exception: + # This opcode is a member of the family but it doesn't pass the checks. + if mac := self.macro_instrs.get("BINARY_OP_INPLACE_ADD_UNICODE"): + mac.family = self.families.get("BINARY_OP") def check_families(self) -> None: """Check each family: diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 0b998d109e9548..01ab83bedb2985 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -781,17 +781,7 @@ def write_instructions( case parsing.Macro(): n_macros += 1 mac = self.macro_instrs[thing.name] - family = self.families.get(mac.name) - if family is None: - for family in self.families.values(): - if mac.name in family.members: - break - else: - family = None - # A sanctioned exception - if mac.name == "BINARY_OP_INPLACE_ADD_UNICODE": - family = self.families["BINARY_OP"] - stacking.write_macro_instr(mac, self.out, family) + stacking.write_macro_instr(mac, self.out) case parsing.Pseudo(): pass case _: diff --git a/Tools/cases_generator/instructions.py b/Tools/cases_generator/instructions.py index 1452ab07390aec..5384bbe81169cc 100644 --- a/Tools/cases_generator/instructions.py +++ b/Tools/cases_generator/instructions.py @@ -278,7 +278,9 @@ class MacroInstruction: macro: parsing.Macro parts: MacroParts cache_offset: int + # Set later predicted: bool = False + family: parsing.Family | None = None @dataclasses.dataclass diff --git a/Tools/cases_generator/stacking.py b/Tools/cases_generator/stacking.py index 32a8d94887ef7e..bba2db8b059da8 100644 --- a/Tools/cases_generator/stacking.py +++ b/Tools/cases_generator/stacking.py @@ -357,9 +357,7 @@ def write_single_instr( raise AssertionError(f"Error writing instruction {instr.name}") from err -def write_macro_instr( - mac: MacroInstruction, out: Formatter, family: Family | None -) -> None: +def write_macro_instr(mac: MacroInstruction, out: Formatter) -> None: parts = [ part for part in mac.parts @@ -369,10 +367,10 @@ def write_macro_instr( with out.block(f"TARGET({mac.name})"): if mac.predicted: out.emit(f"PREDICTED({mac.name});") - out.static_assert_family_size(mac.name, family, mac.cache_offset) + out.static_assert_family_size(mac.name, mac.family, mac.cache_offset) try: next_instr_is_set = write_components( - parts, out, TIER_ONE, mac.cache_offset, family + parts, out, TIER_ONE, mac.cache_offset, mac.family ) except AssertionError as err: raise AssertionError(f"Error writing macro {mac.name}") from err